2024-09-13T13:55:48.285 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T13:55:48.312 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T13:55:48.400 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542 branch: wip-aclamk-cbt-combined description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: akupczyk@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7903542' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-aclamk-cbt-combined ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-aclamk-cbt-combined sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 owner: scheduled_akupczyk@teuthology priority: 160 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 6545 sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 682/13280 suite: upgrade suite_branch: wip-aclamk-cbt-combined suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 targets: smithi017.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE6NWBphV84PWNPdSFPvkZ+KXZyBXoXaqkEpS8qd+x0KvyK/g5nXI1do5KkzBYHQmlxqIAJ3TS09/KAGPSFP5/8= smithi189.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGz+3d6ngbK1ym3b4xAUmBbiVdYujJLh8HvCMrF7OTDZkWJBexBSaRwrN8oDCSdNnV+ZYjn7m5ZgL4K0Imd1bjk= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_10:00:18 tube: smithi user: akupczyk verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T13:55:48.401 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa; will attempt to use it 2024-09-13T13:55:48.401 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks 2024-09-13T13:55:48.402 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T13:55:48.403 INFO:teuthology.task.internal:Checking packages... 2024-09-13T13:55:48.424 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd58f73069a8157a9ace12a0247ee3e8497707ea9' 2024-09-13T13:55:48.424 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T13:55:48.424 INFO:teuthology.packaging:ref: None 2024-09-13T13:55:48.424 INFO:teuthology.packaging:tag: None 2024-09-13T13:55:48.424 INFO:teuthology.packaging:branch: wip-aclamk-cbt-combined 2024-09-13T13:55:48.424 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T13:55:48.424 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-aclamk-cbt-combined 2024-09-13T13:55:48.572 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4697.gd58f7306 2024-09-13T13:55:48.573 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T13:55:48.581 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T13:55:48.581 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T13:55:48.588 INFO:teuthology.task.internal:Saving configuration 2024-09-13T13:55:48.609 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T13:55:48.615 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T13:55:48.648 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi017.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:50:38.699087', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE6NWBphV84PWNPdSFPvkZ+KXZyBXoXaqkEpS8qd+x0KvyK/g5nXI1do5KkzBYHQmlxqIAJ3TS09/KAGPSFP5/8='} 2024-09-13T13:55:48.681 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:50:38.697259', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGz+3d6ngbK1ym3b4xAUmBbiVdYujJLh8HvCMrF7OTDZkWJBexBSaRwrN8oDCSdNnV+ZYjn7m5ZgL4K0Imd1bjk='} 2024-09-13T13:55:48.681 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T13:55:48.686 INFO:teuthology.task.internal:roles: ubuntu@smithi017.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T13:55:48.686 INFO:teuthology.task.internal:roles: ubuntu@smithi189.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T13:55:48.686 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T13:55:48.762 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0da369fbe0>, signals=[15]) 2024-09-13T13:55:48.762 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T13:55:48.767 INFO:teuthology.task.internal:Opening connections... 2024-09-13T13:55:48.767 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi017.front.sepia.ceph.com 2024-09-13T13:55:48.769 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:55:48.853 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi189.front.sepia.ceph.com 2024-09-13T13:55:48.853 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:55:48.933 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T13:55:48.940 DEBUG:teuthology.orchestra.run.smithi017:> uname -m 2024-09-13T13:55:48.958 INFO:teuthology.orchestra.run.smithi017.stdout:x86_64 2024-09-13T13:55:48.958 DEBUG:teuthology.orchestra.run.smithi017:> cat /etc/os-release 2024-09-13T13:55:49.016 INFO:teuthology.orchestra.run.smithi017.stdout:NAME="CentOS Stream" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION="9" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:ID="centos" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:ID_LIKE="rhel fedora" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION_ID="9" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:PLATFORM_ID="platform:el9" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:ANSI_COLOR="0;31" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:LOGO="fedora-logo-icon" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:HOME_URL="https://centos.org/" 2024-09-13T13:55:49.017 INFO:teuthology.orchestra.run.smithi017.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T13:55:49.018 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T13:55:49.018 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T13:55:49.018 INFO:teuthology.lock.ops:Updating smithi017.front.sepia.ceph.com on lock server 2024-09-13T13:55:49.045 DEBUG:teuthology.orchestra.run.smithi189:> uname -m 2024-09-13T13:55:49.063 INFO:teuthology.orchestra.run.smithi189.stdout:x86_64 2024-09-13T13:55:49.063 DEBUG:teuthology.orchestra.run.smithi189:> cat /etc/os-release 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:NAME="CentOS Stream" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION="9" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:ID="centos" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:ID_LIKE="rhel fedora" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_ID="9" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:PLATFORM_ID="platform:el9" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:ANSI_COLOR="0;31" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:LOGO="fedora-logo-icon" 2024-09-13T13:55:49.121 INFO:teuthology.orchestra.run.smithi189.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T13:55:49.122 INFO:teuthology.orchestra.run.smithi189.stdout:HOME_URL="https://centos.org/" 2024-09-13T13:55:49.122 INFO:teuthology.orchestra.run.smithi189.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T13:55:49.122 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T13:55:49.122 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T13:55:49.122 INFO:teuthology.lock.ops:Updating smithi189.front.sepia.ceph.com on lock server 2024-09-13T13:55:49.153 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T13:55:49.162 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T13:55:49.174 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T13:55:49.174 DEBUG:teuthology.orchestra.run.smithi017:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:55:49.177 DEBUG:teuthology.orchestra.run.smithi189:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:55:49.193 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T13:55:49.199 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T13:55:49.199 DEBUG:teuthology.orchestra.run.smithi017:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:55:49.234 DEBUG:teuthology.orchestra.run.smithi189:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:55:49.314 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T13:55:49.383 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T13:55:49.398 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T13:55:49.398 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T13:55:49.398 DEBUG:teuthology.orchestra.run.smithi017:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:55:49.399 DEBUG:teuthology.orchestra.run.smithi189:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:55:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:55:49.416 DEBUG:teuthology.orchestra.run.smithi017:> uname -r 2024-09-13T13:55:49.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:55:49.417 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-09-13T13:55:49.473 INFO:teuthology.orchestra.run.smithi017.stdout:5.14.0-503.el9.x86_64 2024-09-13T13:55:49.473 INFO:teuthology.task.kernel:Running kernel on smithi017: 5.14.0-503.el9.x86_64 2024-09-13T13:55:49.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum install -y kernel 2024-09-13T13:55:49.475 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-503.el9.x86_64 2024-09-13T13:55:49.475 INFO:teuthology.task.kernel:Running kernel on smithi189: 5.14.0-503.el9.x86_64 2024-09-13T13:55:49.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum install -y kernel 2024-09-13T13:55:51.926 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-09-13T13:55:53.027 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-09-13T13:55:54.744 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-13T13:56:02.603 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - CRB 1.6 MB/s | 6.5 MB 00:04 2024-09-13T13:56:04.195 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 19 kB 00:00 2024-09-13T13:56:05.349 INFO:teuthology.orchestra.run.smithi189.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-13T13:56:10.771 INFO:teuthology.orchestra.run.smithi189.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-09-13T13:56:12.184 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 20 MB 00:19 2024-09-13T13:56:12.736 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T13:56:12.736 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T13:56:12.839 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T13:56:12.844 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T13:56:12.844 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout:Install 4 Packages 2024-09-13T13:56:12.845 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:56:12.846 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 79 M 2024-09-13T13:56:12.846 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 126 M 2024-09-13T13:56:12.846 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T13:56:13.056 INFO:teuthology.orchestra.run.smithi189.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 329 kB/s | 23 kB 00:00 2024-09-13T13:56:13.781 INFO:teuthology.orchestra.run.smithi189.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 20 MB/s | 16 MB 00:00 2024-09-13T13:56:14.157 INFO:teuthology.orchestra.run.smithi189.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 30 MB/s | 35 MB 00:01 2024-09-13T13:56:14.490 INFO:teuthology.orchestra.run.smithi189.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 20 MB/s | 29 MB 00:01 2024-09-13T13:56:14.490 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T13:56:14.491 INFO:teuthology.orchestra.run.smithi189.stdout:Total 48 MB/s | 79 MB 00:01 2024-09-13T13:56:15.175 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T13:56:15.556 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T13:56:15.556 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T13:56:16.495 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T13:56:16.495 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T13:56:18.659 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-13T13:56:18.941 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T13:56:19.581 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T13:56:19.674 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:56:21.143 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 19 kB 00:01 2024-09-13T13:56:22.496 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:56:22.531 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 18 MB/s | 23 MB 00:01 2024-09-13T13:56:22.620 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:56:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:56:27.578 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:56:28.164 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-13T13:56:30.227 INFO:teuthology.orchestra.run.smithi017.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T13:56:30.227 INFO:teuthology.orchestra.run.smithi017.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T13:56:30.334 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T13:56:30.344 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T13:56:30.344 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-13T13:56:30.344 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout:Install 4 Packages 2024-09-13T13:56:30.345 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:56:30.346 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 79 M 2024-09-13T13:56:30.346 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 126 M 2024-09-13T13:56:30.346 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T13:56:30.463 INFO:teuthology.orchestra.run.smithi017.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 325 kB/s | 23 kB 00:00 2024-09-13T13:56:31.680 INFO:teuthology.orchestra.run.smithi017.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-13T13:56:31.988 INFO:teuthology.orchestra.run.smithi017.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 22 MB/s | 35 MB 00:01 2024-09-13T13:56:32.407 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:56:32.881 INFO:teuthology.orchestra.run.smithi017.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 12 MB/s | 29 MB 00:02 2024-09-13T13:56:32.881 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T13:56:32.881 INFO:teuthology.orchestra.run.smithi017.stdout:Total 31 MB/s | 79 MB 00:02 2024-09-13T13:56:33.588 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T13:56:33.965 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T13:56:33.965 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T13:56:34.965 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T13:56:34.965 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T13:56:37.534 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T13:56:38.184 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T13:56:38.290 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:56:41.076 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:56:41.200 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:56:46.324 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:56:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:56:51.508 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:10.676 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:10.716 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:10.993 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:10.994 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T13:57:10.994 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:57:10.994 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:57:11.810 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:11.810 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:57:11.810 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T13:57:11.810 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:11.811 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T13:57:11.811 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T13:57:11.811 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T13:57:11.811 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:57:11.811 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T13:57:12.171 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T13:57:12.259 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:12.259 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T13:57:12.260 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T13:57:12.260 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T13:57:12.260 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T13:57:12.260 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T13:57:12.260 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi189.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T13:57:12.260 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T13:57:12.320 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:12.320 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T13:57:12.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo rpm -qi grub2-tools 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Name : grub2-tools 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Epoch : 1 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Version : 2.06 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Release : 82.el9 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: x86_64 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Group : Unspecified 2024-09-13T13:57:12.355 INFO:teuthology.orchestra.run.smithi189.stdout:Size : 8274670 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:License : GPLv3+ 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Packager : builder@centos.org 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Vendor : CentOS 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Summary : Support tools for GRUB. 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:Description : 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T13:57:12.356 INFO:teuthology.orchestra.run.smithi189.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T13:57:12.357 INFO:teuthology.orchestra.run.smithi189.stdout:hardware devices. 2024-09-13T13:57:12.357 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T13:57:12.357 INFO:teuthology.orchestra.run.smithi189.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T13:57:12.357 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T13:57:12.358 INFO:teuthology.task.kernel:Updating grub on smithi189 to boot 5.14.0-505.el9.x86_64 2024-09-13T13:57:12.358 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T13:57:12.985 INFO:teuthology.orchestra.run.smithi189.stderr:Generating grub configuration file ... 2024-09-13T13:57:14.371 INFO:teuthology.orchestra.run.smithi189.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T13:57:14.392 INFO:teuthology.orchestra.run.smithi189.stderr:done 2024-09-13T13:57:14.394 DEBUG:teuthology.orchestra.run.smithi189:> mktemp 2024-09-13T13:57:14.410 INFO:teuthology.orchestra.run.smithi189.stdout:/tmp/tmp.cgTTXN5nJ5 2024-09-13T13:57:14.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.cgTTXN5nJ5 2024-09-13T13:57:14.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0666 /tmp/tmp.cgTTXN5nJ5 2024-09-13T13:57:14.611 DEBUG:teuthology.orchestra.remote:smithi189:/tmp/tmp.cgTTXN5nJ5 is 6KB 2024-09-13T13:57:14.659 DEBUG:teuthology.orchestra.run.smithi189:> rm -fr /tmp/tmp.cgTTXN5nJ5 2024-09-13T13:57:14.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T13:57:14.742 INFO:teuthology.orchestra.run.smithi189.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T13:57:14.742 INFO:teuthology.orchestra.run.smithi189.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T13:57:14.742 INFO:teuthology.orchestra.run.smithi189.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T13:57:14.742 INFO:teuthology.orchestra.run.smithi189.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T13:57:14.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T13:57:14.896 DEBUG:teuthology.orchestra.run.smithi189:> sudo shutdown -r now 2024-09-13T13:57:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:31.025 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:31.025 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T13:57:31.026 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T13:57:31.026 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T13:57:31.890 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T13:57:31.891 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:57:31.891 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T13:57:32.206 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T13:57:32.293 INFO:teuthology.orchestra.run.smithi017.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:32.294 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T13:57:32.294 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T13:57:32.294 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T13:57:32.294 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T13:57:32.294 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T13:57:32.294 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi017.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T13:57:32.294 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T13:57:32.353 INFO:teuthology.orchestra.run.smithi017.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T13:57:32.353 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T13:57:32.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo rpm -qi grub2-tools 2024-09-13T13:57:32.391 INFO:teuthology.orchestra.run.smithi017.stdout:Name : grub2-tools 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Epoch : 1 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Version : 2.06 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Release : 82.el9 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Architecture: x86_64 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Group : Unspecified 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Size : 8274670 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:License : GPLv3+ 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T13:57:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:Packager : builder@centos.org 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:Vendor : CentOS 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:Summary : Support tools for GRUB. 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:Description : 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:hardware devices. 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T13:57:32.393 INFO:teuthology.orchestra.run.smithi017.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T13:57:32.394 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T13:57:32.394 INFO:teuthology.task.kernel:Updating grub on smithi017 to boot 5.14.0-505.el9.x86_64 2024-09-13T13:57:32.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T13:57:33.045 INFO:teuthology.orchestra.run.smithi017.stderr:Generating grub configuration file ... 2024-09-13T13:57:34.378 INFO:teuthology.orchestra.run.smithi017.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T13:57:34.402 INFO:teuthology.orchestra.run.smithi017.stderr:done 2024-09-13T13:57:34.404 DEBUG:teuthology.orchestra.run.smithi017:> mktemp 2024-09-13T13:57:34.419 INFO:teuthology.orchestra.run.smithi017.stdout:/tmp/tmp.eD0inagC99 2024-09-13T13:57:34.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.eD0inagC99 2024-09-13T13:57:34.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 0666 /tmp/tmp.eD0inagC99 2024-09-13T13:57:34.630 DEBUG:teuthology.orchestra.remote:smithi017:/tmp/tmp.eD0inagC99 is 6KB 2024-09-13T13:57:34.678 DEBUG:teuthology.orchestra.run.smithi017:> rm -fr /tmp/tmp.eD0inagC99 2024-09-13T13:57:34.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T13:57:34.755 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T13:57:34.755 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T13:57:34.755 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T13:57:34.755 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T13:57:34.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T13:57:34.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo shutdown -r now 2024-09-13T13:57:44.908 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T13:57:44.908 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T13:57:44.909 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:03.329 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-09-13T13:58:04.929 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T13:58:04.929 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T13:58:04.930 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:12.337 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T13:58:12.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:15.421 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-09-13T13:58:23.293 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.17 2024-09-13T13:58:27.425 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T13:58:27.426 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:32.297 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T13:58:32.297 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:27.489 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T13:59:32.309 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T13:59:42.497 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T13:59:42.498 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:42.878 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-09-13T13:59:43.231 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T13:59:43.231 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T13:59:43.231 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-09-13T13:59:43.247 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-505.el9.x86_64 2024-09-13T13:59:43.248 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T13:59:43.248 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T13:59:43.248 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T13:59:44.249 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T13:59:44.250 DEBUG:teuthology.orchestra.run.smithi189:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T13:59:44.309 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T13:59:44.310 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:44.333 INFO:teuthology.orchestra.run.smithi189.stdout:ttyS1 2024-09-13T13:59:44.354 DEBUG:teuthology.parallel:result is None 2024-09-13T13:59:44.659 DEBUG:teuthology.orchestra.run.smithi017:> true 2024-09-13T13:59:45.001 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T13:59:45.001 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T13:59:45.002 DEBUG:teuthology.orchestra.run.smithi017:> uname -r 2024-09-13T13:59:45.019 INFO:teuthology.orchestra.run.smithi017.stdout:5.14.0-505.el9.x86_64 2024-09-13T13:59:45.019 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T13:59:45.019 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T13:59:45.019 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T13:59:46.020 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T13:59:46.020 DEBUG:teuthology.orchestra.run.smithi017:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T13:59:46.097 INFO:teuthology.orchestra.run.smithi017.stdout:ttyS1 2024-09-13T13:59:46.118 DEBUG:teuthology.parallel:result is None 2024-09-13T13:59:46.118 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T13:59:46.129 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T13:59:46.129 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T13:59:46.133 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T13:59:46.152 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T13:59:46.160 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T13:59:46.169 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T13:59:46.170 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T13:59:46.192 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T13:59:46.223 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T13:59:46.231 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T13:59:46.231 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T13:59:46.258 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T13:59:46.295 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:46.304 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:46.307 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:46.316 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:46.318 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T13:59:46.326 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T13:59:46.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T13:59:46.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T13:59:46.380 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T13:59:46.390 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T13:59:46.390 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T13:59:46.421 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T13:59:46.439 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:46.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:46.574 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:46.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:46.701 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T13:59:46.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T13:59:46.768 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:46.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:46.867 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:46.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:46.995 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T13:59:46.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T13:59:47.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo service rsyslog restart 2024-09-13T13:59:47.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo service rsyslog restart 2024-09-13T13:59:47.105 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T13:59:47.158 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T13:59:47.542 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T13:59:47.550 INFO:teuthology.task.internal:Starting timer... 2024-09-13T13:59:47.550 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T13:59:47.563 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T13:59:47.574 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-13T13:59:47.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo service auditd rotate 2024-09-13T13:59:47.621 INFO:teuthology.orchestra.run.smithi017.stdout:Rotating logs: 2024-09-13T13:59:47.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo service auditd rotate 2024-09-13T13:59:47.669 INFO:teuthology.orchestra.run.smithi189.stdout:Rotating logs: 2024-09-13T13:59:47.671 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T13:59:47.671 DEBUG:teuthology.orchestra.run.smithi017:> /usr/sbin/getenforce 2024-09-13T13:59:47.707 INFO:teuthology.orchestra.run.smithi017.stdout:Permissive 2024-09-13T13:59:47.707 DEBUG:teuthology.orchestra.run.smithi189:> /usr/sbin/getenforce 2024-09-13T13:59:47.735 INFO:teuthology.orchestra.run.smithi189.stdout:Permissive 2024-09-13T13:59:47.735 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi017.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi189.front.sepia.ceph.com': 'permissive'} 2024-09-13T13:59:47.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T13:59:47.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:59:47.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T13:59:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:59:47.804 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T13:59:47.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo /usr/sbin/setenforce permissive 2024-09-13T13:59:47.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo /usr/sbin/setenforce permissive 2024-09-13T13:59:47.887 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T13:59:47.896 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T13:59:47.915 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T13:59:47.917 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi017.front.sepia.ceph.com,smithi189.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T14:03:38.531 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi017.front.sepia.ceph.com'), Remote(name='ubuntu@smithi189.front.sepia.ceph.com')] 2024-09-13T14:03:38.532 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T14:03:38.533 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:03:38.616 DEBUG:teuthology.orchestra.run.smithi017:> true 2024-09-13T14:03:38.692 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-13T14:03:38.693 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T14:03:38.693 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:03:38.777 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-09-13T14:03:38.860 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-09-13T14:03:38.861 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T14:03:38.870 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T14:03:38.870 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:03:38.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:03:38.873 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:03:38.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:03:38.903 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:03:38.919 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:03:38.930 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:03:38.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: ntpd: command not found 2024-09-13T14:03:38.945 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:03:38.956 INFO:teuthology.orchestra.run.smithi017.stdout:506 Cannot talk to daemon 2024-09-13T14:03:38.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: ntpd: command not found 2024-09-13T14:03:38.971 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:03:38.979 INFO:teuthology.orchestra.run.smithi189.stdout:506 Cannot talk to daemon 2024-09-13T14:03:38.986 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:03:38.994 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:03:39.008 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:03:39.042 INFO:teuthology.orchestra.run.smithi017.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.045 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.062 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.066 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:03:39.066 INFO:teuthology.run_tasks:Running task pexec... 2024-09-13T14:03:39.075 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-13T14:03:39.076 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi017.front.sepia.ceph.com 2024-09-13T14:03:39.076 DEBUG:teuthology.orchestra.run.smithi017:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:03:39.076 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi189.front.sepia.ceph.com 2024-09-13T14:03:39.077 DEBUG:teuthology.orchestra.run.smithi189:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:03:39.453 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:03:39.454 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:03:39.456 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:39.456 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-09-13T14:03:39.456 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:39.456 INFO:teuthology.orchestra.run.smithi189.stdout:Removing: 2024-09-13T14:03:39.456 INFO:teuthology.orchestra.run.smithi189.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:Removing unused dependencies: 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:Remove 2 Packages 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:Freed space: 5.5 M 2024-09-13T14:03:39.457 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:03:39.461 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:03:39.461 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:03:39.465 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:39.465 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-13T14:03:39.465 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:39.465 INFO:teuthology.orchestra.run.smithi017.stdout:Removing: 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:Removing unused dependencies: 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:Remove 2 Packages 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:Freed space: 5.5 M 2024-09-13T14:03:39.466 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:03:39.470 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:03:39.470 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:03:39.517 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:03:39.517 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:03:39.540 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:03:39.540 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:03:39.663 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:03:39.693 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:03:39.736 INFO:teuthology.orchestra.run.smithi189.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:03:39.775 INFO:teuthology.orchestra.run.smithi017.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:03:39.802 INFO:teuthology.orchestra.run.smithi189.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:03:39.842 INFO:teuthology.orchestra.run.smithi017.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:03:40.290 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:03:40.290 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:03:40.334 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:03:40.334 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:03:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:03:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:40.575 INFO:teuthology.orchestra.run.smithi017.stdout:Removed: 2024-09-13T14:03:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:03:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:40.576 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout:Removed: 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:40.589 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:03:41.474 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:52 ago on Fri 13 Sep 2024 02:02:49 PM UTC. 2024-09-13T14:03:41.509 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:49 ago on Fri 13 Sep 2024 02:02:52 PM UTC. 2024-09-13T14:03:41.612 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:03:41.613 INFO:teuthology.orchestra.run.smithi189.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout:Install 7 Packages 2024-09-13T14:03:41.614 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:41.646 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 5.2 M 2024-09-13T14:03:41.646 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 19 M 2024-09-13T14:03:41.646 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:03:41.646 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:03:41.647 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:41.647 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:03:41.647 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:41.647 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:03:41.647 INFO:teuthology.orchestra.run.smithi017.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout:Install 7 Packages 2024-09-13T14:03:41.648 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:41.660 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 5.2 M 2024-09-13T14:03:41.661 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 19 M 2024-09-13T14:03:41.661 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:03:42.054 INFO:teuthology.orchestra.run.smithi017.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 382 kB/s | 44 kB 00:00 2024-09-13T14:03:42.096 INFO:teuthology.orchestra.run.smithi017.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 670 kB/s | 106 kB 00:00 2024-09-13T14:03:42.130 INFO:teuthology.orchestra.run.smithi017.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 953 kB/s | 72 kB 00:00 2024-09-13T14:03:42.155 INFO:teuthology.orchestra.run.smithi017.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-13T14:03:42.214 INFO:teuthology.orchestra.run.smithi017.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.2 MB/s | 906 kB 00:00 2024-09-13T14:03:42.306 INFO:teuthology.orchestra.run.smithi017.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.7 MB/s | 837 kB 00:00 2024-09-13T14:03:42.560 INFO:teuthology.orchestra.run.smithi189.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 118 kB/s | 44 kB 00:00 2024-09-13T14:03:42.636 INFO:teuthology.orchestra.run.smithi189.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 234 kB/s | 106 kB 00:00 2024-09-13T14:03:42.753 INFO:teuthology.orchestra.run.smithi189.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 722 kB/s | 84 kB 00:00 2024-09-13T14:03:42.778 INFO:teuthology.orchestra.run.smithi189.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 330 kB/s | 72 kB 00:00 2024-09-13T14:03:42.931 INFO:teuthology.orchestra.run.smithi017.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 4.1 MB/s | 3.2 MB 00:00 2024-09-13T14:03:42.932 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:03:42.932 INFO:teuthology.orchestra.run.smithi017.stdout:Total 4.1 MB/s | 5.2 MB 00:01 2024-09-13T14:03:43.012 INFO:teuthology.orchestra.run.smithi189.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 1.1 MB/s | 906 kB 00:00 2024-09-13T14:03:43.049 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:03:43.065 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:03:43.066 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:03:43.137 INFO:teuthology.orchestra.run.smithi189.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 2.1 MB/s | 837 kB 00:00 2024-09-13T14:03:43.358 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:03:43.359 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:03:43.637 INFO:teuthology.orchestra.run.smithi189.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 3.7 MB/s | 3.2 MB 00:00 2024-09-13T14:03:43.637 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:03:43.637 INFO:teuthology.orchestra.run.smithi189.stdout:Total 2.6 MB/s | 5.2 MB 00:01 2024-09-13T14:03:43.754 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:03:43.769 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:03:43.769 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:03:43.854 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:03:43.951 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:03:44.003 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:03:44.031 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:03:44.116 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:03:44.116 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:03:44.259 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:03:44.295 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:03:44.608 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:03:44.714 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:03:44.756 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:03:44.775 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:03:44.794 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:03:44.797 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:03:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:03:44.996 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:45.007 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:03:45.042 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:03:45.475 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:03:45.511 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:03:45.623 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:03:45.690 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:03:45.986 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:03:46.349 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:03:46.350 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:03:46.673 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:03:46.711 DEBUG:teuthology.parallel:result is None 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:03:47.068 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:03:47.069 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:03:47.454 DEBUG:teuthology.parallel:result is None 2024-09-13T14:03:47.454 INFO:teuthology.run_tasks:Running task install... 2024-09-13T14:03:47.463 DEBUG:teuthology.task.install:project ceph 2024-09-13T14:03:47.463 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:03:47.463 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:03:47.463 INFO:teuthology.task.install:Using flavor: default 2024-09-13T14:03:47.468 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T14:03:47.468 INFO:teuthology.task.install:extra packages: [] 2024-09-13T14:03:47.469 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:03:47.469 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:03:47.469 INFO:teuthology.packaging:ref: None 2024-09-13T14:03:47.469 INFO:teuthology.packaging:tag: None 2024-09-13T14:03:47.469 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:03:47.469 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:03:47.470 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:03:47.471 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:03:47.472 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:03:47.472 INFO:teuthology.packaging:ref: None 2024-09-13T14:03:47.472 INFO:teuthology.packaging:tag: None 2024-09-13T14:03:47.472 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:03:47.472 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:03:47.472 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:03:47.702 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:03:47.702 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:03:47.721 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/ 2024-09-13T14:03:47.721 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1627.gbbdadfa3 2024-09-13T14:03:47.825 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:03:47.826 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:03:47.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:03:47.844 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/bbdadfa31f8fb9afaa389358d138cac49bd69e56/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:03:47.844 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:03:47.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:03:47.851 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:03:47.852 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:03:47.852 INFO:teuthology.packaging:ref: None 2024-09-13T14:03:47.852 INFO:teuthology.packaging:tag: None 2024-09-13T14:03:47.852 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:03:47.852 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:03:47.852 DEBUG:teuthology.orchestra.run.smithi189:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:03:47.876 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:03:47.877 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:03:47.877 INFO:teuthology.packaging:ref: None 2024-09-13T14:03:47.877 INFO:teuthology.packaging:tag: None 2024-09-13T14:03:47.877 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:03:47.877 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:03:47.877 DEBUG:teuthology.orchestra.run.smithi017:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:03:47.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:03:47.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:03:47.993 DEBUG:teuthology.orchestra.run.smithi189:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:03:48.031 DEBUG:teuthology.orchestra.run.smithi017:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:03:48.059 INFO:teuthology.orchestra.run.smithi189.stdout:check_obsoletes = 1 2024-09-13T14:03:48.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum clean all 2024-09-13T14:03:48.099 INFO:teuthology.orchestra.run.smithi017.stdout:check_obsoletes = 1 2024-09-13T14:03:48.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum clean all 2024-09-13T14:03:48.300 INFO:teuthology.orchestra.run.smithi189.stdout:45 files removed 2024-09-13T14:03:48.330 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-radosgw 2024-09-13T14:03:48.334 INFO:teuthology.orchestra.run.smithi017.stdout:45 files removed 2024-09-13T14:03:48.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-radosgw 2024-09-13T14:03:48.893 INFO:teuthology.orchestra.run.smithi189.stdout:ceph packages for x86_64 286 kB/s | 77 kB 00:00 2024-09-13T14:03:48.938 INFO:teuthology.orchestra.run.smithi017.stdout:ceph packages for x86_64 282 kB/s | 77 kB 00:00 2024-09-13T14:03:49.185 INFO:teuthology.orchestra.run.smithi189.stdout:ceph noarch packages 43 kB/s | 11 kB 00:00 2024-09-13T14:03:49.231 INFO:teuthology.orchestra.run.smithi017.stdout:ceph noarch packages 43 kB/s | 11 kB 00:00 2024-09-13T14:03:49.453 INFO:teuthology.orchestra.run.smithi189.stdout:ceph source packages 7.7 kB/s | 1.8 kB 00:00 2024-09-13T14:03:49.506 INFO:teuthology.orchestra.run.smithi017.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-13T14:03:49.953 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - BaseOS 18 MB/s | 8.2 MB 00:00 2024-09-13T14:03:50.831 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - BaseOS 6.4 MB/s | 8.2 MB 00:01 2024-09-13T14:03:53.566 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - AppStream 13 MB/s | 20 MB 00:01 2024-09-13T14:03:55.129 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - AppStream 5.0 MB/s | 20 MB 00:04 2024-09-13T14:03:59.175 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.5 MB 00:01 2024-09-13T14:03:59.588 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-13T14:04:00.918 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 19 kB 00:00 2024-09-13T14:04:01.121 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - Extras packages 96 kB/s | 19 kB 00:00 2024-09-13T14:04:02.622 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 14 MB/s | 23 MB 00:01 2024-09-13T14:04:02.959 INFO:teuthology.orchestra.run.smithi189.stdout:Extra Packages for Enterprise Linux 13 MB/s | 23 MB 00:01 2024-09-13T14:04:08.211 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-13T14:04:08.406 INFO:teuthology.orchestra.run.smithi189.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-13T14:04:10.666 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrading: 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:04:10.668 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:04:10.669 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout:Install 22 Packages 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrade 2 Packages 2024-09-13T14:04:10.670 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:10.671 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 58 M 2024-09-13T14:04:10.671 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:04:10.754 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-radosgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 14 M 2024-09-13T14:04:10.756 INFO:teuthology.orchestra.run.smithi189.stdout:Upgrading: 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: librados2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.1 M 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: librbd1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-base x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 5.0 M 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 22 M 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-selinux x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: libcephfs2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 637 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: libradosstriper1 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 457 k 2024-09-13T14:04:10.757 INFO:teuthology.orchestra.run.smithi189.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: librgw2 x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 4.9 M 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ceph-argparse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 45 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ceph-common x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 120 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cephfs x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 156 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rados x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 320 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 294 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rgw x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 99 k 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:10.758 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout:Install 22 Packages 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout:Upgrade 2 Packages 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 58 M 2024-09-13T14:04:10.759 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:04:11.824 INFO:teuthology.orchestra.run.smithi189.stdout:(1/24): ceph-radosgw-17.2.7-1627.gbbdadfa3.el9. 15 MB/s | 14 MB 00:00 2024-09-13T14:04:11.929 INFO:teuthology.orchestra.run.smithi017.stdout:(1/24): ceph-base-17.2.7-1627.gbbdadfa3.el9.x86 8.1 MB/s | 5.0 MB 00:00 2024-09-13T14:04:11.932 INFO:teuthology.orchestra.run.smithi189.stdout:(2/24): ceph-base-17.2.7-1627.gbbdadfa3.el9.x86 4.7 MB/s | 5.0 MB 00:01 2024-09-13T14:04:11.966 INFO:teuthology.orchestra.run.smithi189.stdout:(3/24): ceph-selinux-17.2.7-1627.gbbdadfa3.el9. 177 kB/s | 25 kB 00:00 2024-09-13T14:04:12.013 INFO:teuthology.orchestra.run.smithi017.stdout:(2/24): ceph-selinux-17.2.7-1627.gbbdadfa3.el9. 302 kB/s | 25 kB 00:00 2024-09-13T14:04:12.075 INFO:teuthology.orchestra.run.smithi189.stdout:(4/24): libradosstriper1-17.2.7-1627.gbbdadfa3. 4.1 MB/s | 457 kB 00:00 2024-09-13T14:04:12.100 INFO:teuthology.orchestra.run.smithi189.stdout:(5/24): libcephfs2-17.2.7-1627.gbbdadfa3.el9.x8 3.7 MB/s | 637 kB 00:00 2024-09-13T14:04:12.130 INFO:teuthology.orchestra.run.smithi017.stdout:(3/24): libcephfs2-17.2.7-1627.gbbdadfa3.el9.x8 5.3 MB/s | 637 kB 00:00 2024-09-13T14:04:12.225 INFO:teuthology.orchestra.run.smithi189.stdout:(6/24): python3-ceph-argparse-17.2.7-1627.gbbda 361 kB/s | 45 kB 00:00 2024-09-13T14:04:12.246 INFO:teuthology.orchestra.run.smithi017.stdout:(4/24): libradosstriper1-17.2.7-1627.gbbdadfa3. 3.8 MB/s | 457 kB 00:00 2024-09-13T14:04:12.351 INFO:teuthology.orchestra.run.smithi189.stdout:(7/24): python3-ceph-common-17.2.7-1627.gbbdadf 961 kB/s | 120 kB 00:00 2024-09-13T14:04:12.551 INFO:teuthology.orchestra.run.smithi189.stdout:(8/24): python3-cephfs-17.2.7-1627.gbbdadfa3.el 780 kB/s | 156 kB 00:00 2024-09-13T14:04:12.614 INFO:teuthology.orchestra.run.smithi017.stdout:(5/24): ceph-radosgw-17.2.7-1627.gbbdadfa3.el9. 11 MB/s | 14 MB 00:01 2024-09-13T14:04:12.672 INFO:teuthology.orchestra.run.smithi017.stdout:(6/24): python3-ceph-argparse-17.2.7-1627.gbbda 777 kB/s | 45 kB 00:00 2024-09-13T14:04:12.685 INFO:teuthology.orchestra.run.smithi189.stdout:(9/24): librgw2-17.2.7-1627.gbbdadfa3.el9.x86_6 8.1 MB/s | 4.9 MB 00:00 2024-09-13T14:04:12.739 INFO:teuthology.orchestra.run.smithi017.stdout:(7/24): python3-ceph-common-17.2.7-1627.gbbdadf 1.8 MB/s | 120 kB 00:00 2024-09-13T14:04:12.760 INFO:teuthology.orchestra.run.smithi189.stdout:(10/24): python3-rados-17.2.7-1627.gbbdadfa3.el 1.5 MB/s | 320 kB 00:00 2024-09-13T14:04:12.785 INFO:teuthology.orchestra.run.smithi189.stdout:(11/24): python3-rbd-17.2.7-1627.gbbdadfa3.el9. 2.9 MB/s | 294 kB 00:00 2024-09-13T14:04:12.810 INFO:teuthology.orchestra.run.smithi189.stdout:(12/24): python3-rgw-17.2.7-1627.gbbdadfa3.el9. 1.9 MB/s | 99 kB 00:00 2024-09-13T14:04:12.814 INFO:teuthology.orchestra.run.smithi017.stdout:(8/24): librgw2-17.2.7-1627.gbbdadfa3.el9.x86_6 8.7 MB/s | 4.9 MB 00:00 2024-09-13T14:04:12.840 INFO:teuthology.orchestra.run.smithi017.stdout:(9/24): python3-cephfs-17.2.7-1627.gbbdadfa3.el 1.5 MB/s | 156 kB 00:00 2024-09-13T14:04:12.865 INFO:teuthology.orchestra.run.smithi017.stdout:(10/24): python3-rados-17.2.7-1627.gbbdadfa3.el 6.2 MB/s | 320 kB 00:00 2024-09-13T14:04:12.890 INFO:teuthology.orchestra.run.smithi017.stdout:(11/24): python3-rbd-17.2.7-1627.gbbdadfa3.el9. 5.7 MB/s | 294 kB 00:00 2024-09-13T14:04:12.911 INFO:teuthology.orchestra.run.smithi189.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-09-13T14:04:12.915 INFO:teuthology.orchestra.run.smithi017.stdout:(12/24): python3-rgw-17.2.7-1627.gbbdadfa3.el9. 1.9 MB/s | 99 kB 00:00 2024-09-13T14:04:13.128 INFO:teuthology.orchestra.run.smithi189.stdout:(14/24): ceph-common-17.2.7-1627.gbbdadfa3.el9. 10 MB/s | 22 MB 00:02 2024-09-13T14:04:13.141 INFO:teuthology.orchestra.run.smithi017.stdout:(13/24): ceph-common-17.2.7-1627.gbbdadfa3.el9. 12 MB/s | 22 MB 00:01 2024-09-13T14:04:13.153 INFO:teuthology.orchestra.run.smithi189.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 132 kB/s | 45 kB 00:00 2024-09-13T14:04:13.178 INFO:teuthology.orchestra.run.smithi189.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.4 MB/s | 662 kB 00:00 2024-09-13T14:04:13.237 INFO:teuthology.orchestra.run.smithi189.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-13T14:04:13.270 INFO:teuthology.orchestra.run.smithi189.stdout:(18/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 3.3 MB/s | 308 kB 00:00 2024-09-13T14:04:13.296 INFO:teuthology.orchestra.run.smithi189.stdout:(19/24): libunwind-1.6.2-1.el9.x86_64.rpm 2.8 MB/s | 67 kB 00:00 2024-09-13T14:04:13.321 INFO:teuthology.orchestra.run.smithi189.stdout:(20/24): python3-prettytable-0.7.2-27.el9.noarc 252 kB/s | 42 kB 00:00 2024-09-13T14:04:13.346 INFO:teuthology.orchestra.run.smithi189.stdout:(21/24): liboath-2.6.7-2.el9.x86_64.rpm 446 kB/s | 49 kB 00:00 2024-09-13T14:04:13.441 INFO:teuthology.orchestra.run.smithi017.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 86 kB/s | 45 kB 00:00 2024-09-13T14:04:13.446 INFO:teuthology.orchestra.run.smithi189.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-09-13T14:04:13.588 INFO:teuthology.orchestra.run.smithi189.stdout:(23/24): librbd1-17.2.7-1627.gbbdadfa3.el9.x86_ 12 MB/s | 2.9 MB 00:00 2024-09-13T14:04:13.633 INFO:teuthology.orchestra.run.smithi017.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 216 kB/s | 160 kB 00:00 2024-09-13T14:04:13.638 INFO:teuthology.orchestra.run.smithi189.stdout:(24/24): librados2-17.2.7-1627.gbbdadfa3.el9.x8 11 MB/s | 3.1 MB 00:00 2024-09-13T14:04:13.642 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:13.643 INFO:teuthology.orchestra.run.smithi189.stdout:Total 20 MB/s | 58 MB 00:02 2024-09-13T14:04:13.705 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:04:13.739 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:04:13.739 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:04:13.758 INFO:teuthology.orchestra.run.smithi017.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 339 kB/s | 42 kB 00:00 2024-09-13T14:04:13.809 INFO:teuthology.orchestra.run.smithi017.stdout:(17/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 6.0 MB/s | 308 kB 00:00 2024-09-13T14:04:13.834 INFO:teuthology.orchestra.run.smithi017.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-13T14:04:13.859 INFO:teuthology.orchestra.run.smithi017.stdout:(19/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 700 kB/s | 292 kB 00:00 2024-09-13T14:04:13.885 INFO:teuthology.orchestra.run.smithi017.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-13T14:04:13.960 INFO:teuthology.orchestra.run.smithi017.stdout:(21/24): librdkafka-1.6.1-102.el9.x86_64.rpm 809 kB/s | 662 kB 00:00 2024-09-13T14:04:14.011 INFO:teuthology.orchestra.run.smithi017.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-09-13T14:04:14.169 INFO:teuthology.orchestra.run.smithi017.stdout:(23/24): librbd1-17.2.7-1627.gbbdadfa3.el9.x86_ 14 MB/s | 2.9 MB 00:00 2024-09-13T14:04:14.209 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:04:14.209 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:04:14.227 INFO:teuthology.orchestra.run.smithi017.stdout:(24/24): librados2-17.2.7-1627.gbbdadfa3.el9.x8 9.2 MB/s | 3.1 MB 00:00 2024-09-13T14:04:14.232 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:14.232 INFO:teuthology.orchestra.run.smithi017.stdout:Total 16 MB/s | 58 MB 00:03 2024-09-13T14:04:14.297 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:04:14.335 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:04:14.335 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:04:14.828 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:04:14.829 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:04:15.392 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:04:15.727 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-13T14:04:15.832 INFO:teuthology.orchestra.run.smithi189.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:15.930 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:16.020 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:04:16.042 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:16.047 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:04:16.131 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:16.178 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-13T14:04:16.221 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-13T14:04:16.453 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-13T14:04:16.513 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:16.610 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:16.665 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-13T14:04:16.694 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:04:16.712 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:04:16.748 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:16.757 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:04:16.794 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-13T14:04:16.821 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:16.829 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 10/26 2024-09-13T14:04:16.865 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:04:16.875 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-13T14:04:16.904 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-13T14:04:16.928 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:04:16.955 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:04:17.020 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:04:17.059 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-13T14:04:17.105 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-13T14:04:17.138 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-13T14:04:17.390 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-13T14:04:17.426 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:04:17.430 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-13T14:04:17.475 INFO:teuthology.orchestra.run.smithi189.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:04:17.483 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 8/26 2024-09-13T14:04:17.520 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-13T14:04:17.563 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 10/26 2024-09-13T14:04:17.565 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:04:17.600 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:04:17.639 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 19/26 2024-09-13T14:04:17.663 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:04:17.669 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 20/26 2024-09-13T14:04:17.689 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:04:17.755 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 13/26 2024-09-13T14:04:17.812 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-13T14:04:17.848 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-13T14:04:17.881 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-13T14:04:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-13T14:04:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:04:18.309 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 18/26 2024-09-13T14:04:18.384 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 19/26 2024-09-13T14:04:18.420 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 20/26 2024-09-13T14:04:20.097 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:20.132 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:20.676 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:20.719 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:04:20.803 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:04:20.803 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:04:20.804 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:04:20.804 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:20.836 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:20.929 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:20.968 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:21.567 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 21/26 2024-09-13T14:04:21.613 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:04:21.689 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 22/26 2024-09-13T14:04:21.690 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:04:21.690 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:04:21.690 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:21.775 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:30.764 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /sys 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /proc 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /mnt 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /var/tmp 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /home 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /root 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout:skipping the directory /tmp 2024-09-13T14:04:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /sys 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /proc 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /mnt 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /var/tmp 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /home 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /root 2024-09-13T14:04:32.039 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /tmp 2024-09-13T14:04:32.040 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:32.078 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:04:32.148 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:32.150 INFO:teuthology.orchestra.run.smithi189.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:04:32.230 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:04:32.231 INFO:teuthology.orchestra.run.smithi189.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/26 2024-09-13T14:04:33.458 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 6/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 8/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 9/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 13/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-13T14:04:33.459 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-13T14:04:33.460 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/26 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 24/26 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:04:33.486 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:33.488 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:04:33.584 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-13T14:04:33.585 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:34.237 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:34.237 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout:Upgraded: 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:04:34.238 INFO:teuthology.orchestra.run.smithi189.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.239 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:34.240 INFO:teuthology.orchestra.run.smithi189.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:04:34.240 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:34.240 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:04:34.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-test 2024-09-13T14:04:34.882 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_6 6/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 7/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9. 8/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x8 9/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 10/26 2024-09-13T14:04:34.883 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 11/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 12/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 13/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-13T14:04:34.884 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-13T14:04:34.885 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 23/26 2024-09-13T14:04:34.885 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-13T14:04:34.885 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 25/26 2024-09-13T14:04:35.165 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:27 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:04:35.298 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout:Install 4 Packages 2024-09-13T14:04:35.299 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:35.300 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 48 M 2024-09-13T14:04:35.300 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 193 M 2024-09-13T14:04:35.300 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:04:35.725 INFO:teuthology.orchestra.run.smithi189.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.0 MB/s | 304 kB 00:00 2024-09-13T14:04:35.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-13T14:04:35.785 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:35.785 INFO:teuthology.orchestra.run.smithi017.stdout:Upgraded: 2024-09-13T14:04:35.785 INFO:teuthology.orchestra.run.smithi017.stdout: librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.786 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:35.787 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:04:35.800 INFO:teuthology.orchestra.run.smithi189.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 851 kB/s | 64 kB 00:00 2024-09-13T14:04:35.825 INFO:teuthology.orchestra.run.smithi189.stdout:(3/4): libcephsqlite-17.2.7-1627.gbbdadfa3.el9. 658 kB/s | 163 kB 00:00 2024-09-13T14:04:36.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-test 2024-09-13T14:04:36.723 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:28 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:36.866 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 48 M 2024-09-13T14:04:36.867 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 163 k 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout:Install 4 Packages 2024-09-13T14:04:36.868 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:36.869 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 48 M 2024-09-13T14:04:36.869 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 193 M 2024-09-13T14:04:36.869 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:04:37.146 INFO:teuthology.orchestra.run.smithi017.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.4 MB/s | 304 kB 00:00 2024-09-13T14:04:37.171 INFO:teuthology.orchestra.run.smithi017.stdout:(2/4): libcephsqlite-17.2.7-1627.gbbdadfa3.el9. 666 kB/s | 163 kB 00:00 2024-09-13T14:04:37.196 INFO:teuthology.orchestra.run.smithi017.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-13T14:04:37.809 INFO:teuthology.orchestra.run.smithi189.stdout:(4/4): ceph-test-17.2.7-1627.gbbdadfa3.el9.x86_ 21 MB/s | 48 MB 00:02 2024-09-13T14:04:37.811 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:37.811 INFO:teuthology.orchestra.run.smithi189.stdout:Total 19 MB/s | 48 MB 00:02 2024-09-13T14:04:37.839 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:04:37.858 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:04:37.859 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:04:38.228 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:04:38.229 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:04:38.522 INFO:teuthology.orchestra.run.smithi017.stdout:(4/4): ceph-test-17.2.7-1627.gbbdadfa3.el9.x86_ 30 MB/s | 48 MB 00:01 2024-09-13T14:04:38.524 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:38.524 INFO:teuthology.orchestra.run.smithi017.stdout:Total 29 MB/s | 48 MB 00:01 2024-09-13T14:04:38.553 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:04:38.574 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:04:38.574 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:04:38.776 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:04:38.873 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:04:38.924 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:04:38.955 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:04:38.961 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:04:38.962 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:04:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:04:39.708 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:04:39.773 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:04:39.802 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:04:43.339 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:04:43.411 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:04:44.571 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/4 2024-09-13T14:04:44.653 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:04:45.610 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:04:45.610 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/4 2024-09-13T14:04:45.610 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/4 2024-09-13T14:04:45.610 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:04:45.917 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:45.918 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:04:46.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph 2024-09-13T14:04:46.786 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:38 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:46.881 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/4 2024-09-13T14:04:46.882 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/4 2024-09-13T14:04:46.882 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/4 2024-09-13T14:04:46.882 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:04:46.922 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:04:46.924 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:46.924 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:04:46.924 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:46.924 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:04:46.925 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-collections 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:04:46.926 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:46.927 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout:Install 34 Packages 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 27 M 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 96 M 2024-09-13T14:04:46.928 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:04:47.187 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:47.188 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:04:47.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph 2024-09-13T14:04:47.553 INFO:teuthology.orchestra.run.smithi189.stdout:(1/34): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64.r 32 kB/s | 6.5 kB 00:00 2024-09-13T14:04:47.653 INFO:teuthology.orchestra.run.smithi189.stdout:(2/34): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86_ 6.8 MB/s | 2.1 MB 00:00 2024-09-13T14:04:47.712 INFO:teuthology.orchestra.run.smithi189.stdout:(3/34): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86_ 3.9 MB/s | 1.4 MB 00:00 2024-09-13T14:04:48.057 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:40 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:48.200 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:04:48.204 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:48.204 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:04:48.204 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 6.5 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.1 M 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 1.4 M 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 240 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 3.9 M 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 15 M 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:04:48.205 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-collections 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:04:48.206 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:04:48.207 INFO:teuthology.orchestra.run.smithi017.stdout:Install 34 Packages 2024-09-13T14:04:48.208 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:48.208 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 27 M 2024-09-13T14:04:48.208 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 96 M 2024-09-13T14:04:48.208 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:04:48.287 INFO:teuthology.orchestra.run.smithi189.stdout:(4/34): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x86_ 24 MB/s | 15 MB 00:00 2024-09-13T14:04:48.312 INFO:teuthology.orchestra.run.smithi189.stdout:(5/34): ceph-mgr-modules-core-17.2.7-1627.gbbda 400 kB/s | 240 kB 00:00 2024-09-13T14:04:48.379 INFO:teuthology.orchestra.run.smithi189.stdout:(6/34): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86_ 4.7 MB/s | 3.9 MB 00:00 2024-09-13T14:04:48.463 INFO:teuthology.orchestra.run.smithi189.stdout:(7/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-09-13T14:04:48.476 INFO:teuthology.orchestra.run.smithi017.stdout:(1/34): ceph-17.2.7-1627.gbbdadfa3.el9.x86_64.r 49 kB/s | 6.5 kB 00:00 2024-09-13T14:04:48.505 INFO:teuthology.orchestra.run.smithi189.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 851 kB/s | 106 kB 00:00 2024-09-13T14:04:48.530 INFO:teuthology.orchestra.run.smithi189.stdout:(9/34): python3-pycparser-2.20-6.el9.noarch.rpm 2.0 MB/s | 135 kB 00:00 2024-09-13T14:04:48.572 INFO:teuthology.orchestra.run.smithi189.stdout:(10/34): python3-cryptography-36.0.1-4.el9.x86_ 4.7 MB/s | 1.2 MB 00:00 2024-09-13T14:04:48.605 INFO:teuthology.orchestra.run.smithi189.stdout:(11/34): python3-requests-2.25.1-8.el9.noarch.r 1.2 MB/s | 125 kB 00:00 2024-09-13T14:04:48.630 INFO:teuthology.orchestra.run.smithi189.stdout:(12/34): python3-urllib3-1.26.5-6.el9.noarch.rp 2.1 MB/s | 215 kB 00:00 2024-09-13T14:04:48.643 INFO:teuthology.orchestra.run.smithi017.stdout:(2/34): ceph-mds-17.2.7-1627.gbbdadfa3.el9.x86_ 6.9 MB/s | 2.1 MB 00:00 2024-09-13T14:04:48.693 INFO:teuthology.orchestra.run.smithi017.stdout:(3/34): ceph-mgr-17.2.7-1627.gbbdadfa3.el9.x86_ 4.0 MB/s | 1.4 MB 00:00 2024-09-13T14:04:48.786 INFO:teuthology.orchestra.run.smithi017.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1627.gbbda 2.6 MB/s | 240 kB 00:00 2024-09-13T14:04:48.852 INFO:teuthology.orchestra.run.smithi017.stdout:(5/34): ceph-mon-17.2.7-1627.gbbdadfa3.el9.x86_ 10 MB/s | 3.9 MB 00:00 2024-09-13T14:04:48.897 INFO:teuthology.orchestra.run.smithi189.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 119 kB/s | 35 kB 00:00 2024-09-13T14:04:49.023 INFO:teuthology.orchestra.run.smithi189.stdout:(14/34): python3-packaging-20.9-5.el9.noarch.rp 197 kB/s | 77 kB 00:00 2024-09-13T14:04:49.106 INFO:teuthology.orchestra.run.smithi189.stdout:(15/34): python3-pytz-2021.1-5.el9.noarch.rpm 244 kB/s | 51 kB 00:00 2024-09-13T14:04:49.140 INFO:teuthology.orchestra.run.smithi189.stdout:(16/34): python3-toml-0.10.2-6.el9.noarch.rpm 356 kB/s | 42 kB 00:00 2024-09-13T14:04:49.153 INFO:teuthology.orchestra.run.smithi017.stdout:(6/34): ceph-osd-17.2.7-1627.gbbdadfa3.el9.x86_ 30 MB/s | 15 MB 00:00 2024-09-13T14:04:49.173 INFO:teuthology.orchestra.run.smithi189.stdout:(17/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 650 kB/s | 43 kB 00:00 2024-09-13T14:04:49.215 INFO:teuthology.orchestra.run.smithi189.stdout:(18/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.3 MB/s | 172 kB 00:00 2024-09-13T14:04:49.240 INFO:teuthology.orchestra.run.smithi189.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 432 kB/s | 11 kB 00:00 2024-09-13T14:04:49.266 INFO:teuthology.orchestra.run.smithi189.stdout:(20/34): python3-mako-1.1.4-6.el9.noarch.rpm 248 kB/s | 172 kB 00:00 2024-09-13T14:04:49.283 INFO:teuthology.orchestra.run.smithi189.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 420 kB/s | 18 kB 00:00 2024-09-13T14:04:49.316 INFO:teuthology.orchestra.run.smithi189.stdout:(22/34): python3-cherrypy-18.6.1-2.el9.noarch.r 2.5 MB/s | 358 kB 00:00 2024-09-13T14:04:49.333 INFO:teuthology.orchestra.run.smithi189.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 455 kB/s | 19 kB 00:00 2024-09-13T14:04:49.350 INFO:teuthology.orchestra.run.smithi189.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 621 kB/s | 20 kB 00:00 2024-09-13T14:04:49.367 INFO:teuthology.orchestra.run.smithi189.stdout:(25/34): python3-jaraco-collections-3.0.0-8.el9 229 kB/s | 23 kB 00:00 2024-09-13T14:04:49.384 INFO:teuthology.orchestra.run.smithi189.stdout:(26/34): python3-logutils-0.3.5-21.el9.noarch.r 922 kB/s | 46 kB 00:00 2024-09-13T14:04:49.409 INFO:teuthology.orchestra.run.smithi189.stdout:(27/34): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-13T14:04:49.443 INFO:teuthology.orchestra.run.smithi189.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 281 kB/s | 16 kB 00:00 2024-09-13T14:04:49.468 INFO:teuthology.orchestra.run.smithi189.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.5 MB/s | 90 kB 00:00 2024-09-13T14:04:49.478 INFO:teuthology.orchestra.run.smithi017.stdout:(7/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 366 kB/s | 253 kB 00:00 2024-09-13T14:04:49.485 INFO:teuthology.orchestra.run.smithi189.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 852 kB/s | 36 kB 00:00 2024-09-13T14:04:49.510 INFO:teuthology.orchestra.run.smithi189.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.9 MB/s | 272 kB 00:00 2024-09-13T14:04:49.536 INFO:teuthology.orchestra.run.smithi189.stdout:(32/34): python3-webob-1.8.8-2.el9.noarch.rpm 3.4 MB/s | 230 kB 00:00 2024-09-13T14:04:49.553 INFO:teuthology.orchestra.run.smithi189.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 477 kB/s | 20 kB 00:00 2024-09-13T14:04:49.603 INFO:teuthology.orchestra.run.smithi189.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.6 MB/s | 427 kB 00:00 2024-09-13T14:04:49.607 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:49.608 INFO:teuthology.orchestra.run.smithi189.stdout:Total 10 MB/s | 27 MB 00:02 2024-09-13T14:04:49.630 INFO:teuthology.orchestra.run.smithi017.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 223 kB/s | 106 kB 00:00 2024-09-13T14:04:49.680 INFO:teuthology.orchestra.run.smithi017.stdout:(9/34): python3-pycparser-2.20-6.el9.noarch.rpm 669 kB/s | 135 kB 00:00 2024-09-13T14:04:49.744 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:04:49.782 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:04:49.782 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:04:49.831 INFO:teuthology.orchestra.run.smithi017.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 623 kB/s | 125 kB 00:00 2024-09-13T14:04:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:(11/34): python3-cryptography-36.0.1-4.el9.x86_ 1.2 MB/s | 1.2 MB 00:01 2024-09-13T14:04:49.898 INFO:teuthology.orchestra.run.smithi017.stdout:(12/34): python3-urllib3-1.26.5-6.el9.noarch.rp 989 kB/s | 215 kB 00:00 2024-09-13T14:04:49.973 INFO:teuthology.orchestra.run.smithi017.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 346 kB/s | 35 kB 00:00 2024-09-13T14:04:50.007 INFO:teuthology.orchestra.run.smithi017.stdout:(14/34): python3-mako-1.1.4-6.el9.noarch.rpm 977 kB/s | 172 kB 00:00 2024-09-13T14:04:50.032 INFO:teuthology.orchestra.run.smithi017.stdout:(15/34): python3-packaging-20.9-5.el9.noarch.rp 575 kB/s | 77 kB 00:00 2024-09-13T14:04:50.057 INFO:teuthology.orchestra.run.smithi017.stdout:(16/34): python3-pytz-2021.1-5.el9.noarch.rpm 607 kB/s | 51 kB 00:00 2024-09-13T14:04:50.083 INFO:teuthology.orchestra.run.smithi017.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 552 kB/s | 42 kB 00:00 2024-09-13T14:04:50.150 INFO:teuthology.orchestra.run.smithi017.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 371 kB/s | 43 kB 00:00 2024-09-13T14:04:50.173 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:04:50.174 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:04:50.175 INFO:teuthology.orchestra.run.smithi017.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 463 kB/s | 11 kB 00:00 2024-09-13T14:04:50.200 INFO:teuthology.orchestra.run.smithi017.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.2 MB/s | 172 kB 00:00 2024-09-13T14:04:50.225 INFO:teuthology.orchestra.run.smithi017.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-09-13T14:04:50.251 INFO:teuthology.orchestra.run.smithi017.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 463 kB/s | 23 kB 00:00 2024-09-13T14:04:50.276 INFO:teuthology.orchestra.run.smithi017.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-09-13T14:04:50.301 INFO:teuthology.orchestra.run.smithi017.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-09-13T14:04:50.327 INFO:teuthology.orchestra.run.smithi017.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 920 kB/s | 46 kB 00:00 2024-09-13T14:04:50.352 INFO:teuthology.orchestra.run.smithi017.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-13T14:04:50.377 INFO:teuthology.orchestra.run.smithi017.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-09-13T14:04:50.402 INFO:teuthology.orchestra.run.smithi017.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 327 kB/s | 16 kB 00:00 2024-09-13T14:04:50.428 INFO:teuthology.orchestra.run.smithi017.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-13T14:04:50.453 INFO:teuthology.orchestra.run.smithi017.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 714 kB/s | 36 kB 00:00 2024-09-13T14:04:50.478 INFO:teuthology.orchestra.run.smithi017.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-13T14:04:50.503 INFO:teuthology.orchestra.run.smithi017.stdout:(32/34): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-09-13T14:04:50.528 INFO:teuthology.orchestra.run.smithi017.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 400 kB/s | 20 kB 00:00 2024-09-13T14:04:50.562 INFO:teuthology.orchestra.run.smithi017.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-13T14:04:50.567 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:04:50.567 INFO:teuthology.orchestra.run.smithi017.stdout:Total 12 MB/s | 27 MB 00:02 2024-09-13T14:04:50.706 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:04:50.748 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:04:50.748 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:04:51.157 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:04:51.157 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:04:51.389 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:04:51.464 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-13T14:04:51.496 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-13T14:04:51.529 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-13T14:04:51.621 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-13T14:04:51.715 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-13T14:04:51.756 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-13T14:04:51.800 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-13T14:04:51.846 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-13T14:04:51.881 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-13T14:04:51.914 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-13T14:04:51.954 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-13T14:04:51.989 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-13T14:04:52.024 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-13T14:04:52.072 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-13T14:04:52.104 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:04:52.177 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-13T14:04:52.316 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-13T14:04:52.404 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-13T14:04:52.457 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-13T14:04:52.475 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:04:52.495 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-13T14:04:52.540 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-13T14:04:52.544 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-13T14:04:52.573 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-13T14:04:52.599 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-13T14:04:52.614 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-13T14:04:52.711 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-13T14:04:52.789 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-13T14:04:52.796 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-13T14:04:52.832 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-13T14:04:52.843 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-13T14:04:52.877 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-13T14:04:52.902 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-13T14:04:52.907 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-13T14:04:52.941 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-13T14:04:52.983 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-13T14:04:52.991 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-13T14:04:53.024 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-13T14:04:53.031 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-13T14:04:53.068 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-13T14:04:53.101 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-13T14:04:53.143 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-13T14:04:53.148 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-13T14:04:53.182 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:04:53.260 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-13T14:04:53.293 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 29/34 2024-09-13T14:04:53.325 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:04:53.350 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:04:53.351 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:53.351 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:04:53.351 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:04:53.351 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:04:53.351 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:53.414 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-13T14:04:53.503 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-13T14:04:53.555 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-13T14:04:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-13T14:04:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-13T14:04:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-13T14:04:53.904 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-13T14:04:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-13T14:04:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-13T14:04:54.143 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-13T14:04:54.187 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-13T14:04:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-13T14:04:54.465 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 29/34 2024-09-13T14:04:54.488 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:04:54.516 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 30/34 2024-09-13T14:04:54.516 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:54.516 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:04:54.516 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:04:54.517 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:04:54.517 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:54.709 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:04:54.739 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:04:54.740 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:54.740 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:04:54.740 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:04:54.740 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:04:54.740 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:55.110 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:04:55.135 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:55.336 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:04:55.360 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:04:55.360 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:55.360 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:04:55.360 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:04:55.360 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:04:55.361 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:55.424 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:04:55.914 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:04:55.943 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 31/34 2024-09-13T14:04:55.943 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:55.944 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:04:55.944 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:04:55.944 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:04:55.944 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:56.340 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 32/34 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:04:56.365 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:04:56.599 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 33/34 2024-09-13T14:04:56.599 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:04:56.599 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:04:56.599 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:04:56.600 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:04:56.600 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:56.657 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:04:57.018 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 6/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-13T14:04:57.019 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-13T14:04:57.020 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-13T14:04:57.021 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-13T14:04:57.022 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-13T14:04:58.038 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-13T14:04:58.038 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:04:58.039 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:04:58.040 INFO:teuthology.orchestra.run.smithi189.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:58.041 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 34/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 3/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 4/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 5/34 2024-09-13T14:04:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9. 6/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-13T14:04:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-13T14:04:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-13T14:04:58.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-base 2024-09-13T14:04:59.006 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:51 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:59.098 INFO:teuthology.orchestra.run.smithi189.stdout:Package ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:04:59.141 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:04:59.142 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:04:59.142 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:04:59.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install cephadm 2024-09-13T14:04:59.343 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:04:59.344 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-13T14:04:59.345 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:04:59.346 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:04:59.347 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:04:59.347 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:04:59.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-base 2024-09-13T14:04:59.778 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:51 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:04:59.912 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:04:59.913 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 83 k 2024-09-13T14:04:59.914 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 375 k 2024-09-13T14:04:59.914 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:00.103 INFO:teuthology.orchestra.run.smithi189.stdout:cephadm-17.2.7-1627.gbbdadfa3.el9.noarch.rpm 436 kB/s | 83 kB 00:00 2024-09-13T14:05:00.104 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:00.104 INFO:teuthology.orchestra.run.smithi189.stdout:Total 433 kB/s | 83 kB 00:00 2024-09-13T14:05:00.104 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:00.109 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:00.109 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:00.111 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:00.111 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:00.189 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:00.325 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:52 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:00.419 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-base-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:00.464 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:00.465 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:00.465 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:00.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install cephadm 2024-09-13T14:05:00.643 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:00.740 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:01.079 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:01.139 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:53 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:01.278 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:01.279 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:01.279 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:01.279 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 83 k 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 83 k 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 375 k 2024-09-13T14:05:01.280 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:01.319 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:01.473 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm-17.2.7-1627.gbbdadfa3.el9.noarch.rpm 429 kB/s | 83 kB 00:00 2024-09-13T14:05:01.474 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:01.474 INFO:teuthology.orchestra.run.smithi017.stdout:Total 425 kB/s | 83 kB 00:00 2024-09-13T14:05:01.474 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:01.480 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:01.480 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:01.482 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:01.482 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:01.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:05:01.562 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:01.983 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:02.100 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:02.112 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:54 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:02.251 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:02.251 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:02.251 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repo Size 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:05:02.252 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:02.253 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 137 k 2024-09-13T14:05:02.253 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 379 k 2024-09-13T14:05:02.253 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:02.454 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:02.483 INFO:teuthology.orchestra.run.smithi189.stdout:ceph-immutable-object-cache-17.2.7-1627.gbbdadf 596 kB/s | 137 kB 00:00 2024-09-13T14:05:02.484 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:02.484 INFO:teuthology.orchestra.run.smithi189.stdout:Total 592 kB/s | 137 kB 00:00 2024-09-13T14:05:02.484 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:02.492 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:02.492 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:02.551 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:02.551 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/1 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:02.693 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:02.694 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:02.751 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:02.777 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:02.777 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:05:02.777 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:05:02.777 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:02.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:05:03.503 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:55 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:03.641 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 137 k 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:03.643 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 137 k 2024-09-13T14:05:03.643 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 379 k 2024-09-13T14:05:03.643 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:03.697 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:03.846 INFO:teuthology.orchestra.run.smithi017.stdout:ceph-immutable-object-cache-17.2.7-1627.gbbdadf 678 kB/s | 137 kB 00:00 2024-09-13T14:05:03.847 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:03.847 INFO:teuthology.orchestra.run.smithi017.stdout:Total 672 kB/s | 137 kB 00:00 2024-09-13T14:05:03.847 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:03.855 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:03.855 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:03.911 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-mgr 2024-09-13T14:05:03.916 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:03.917 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:04.174 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:04.174 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:05:04.174 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:05:04.174 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:04.498 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:56 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:04.590 INFO:teuthology.orchestra.run.smithi189.stdout:Package ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:04.633 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:04.636 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:04.636 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:04.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3. 1/1 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:05.131 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:05.282 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:00:57 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:05.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr 2024-09-13T14:05:05.415 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:05.416 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:05.416 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:05:05.416 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:Installing weak dependencies: 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:05.417 INFO:teuthology.orchestra.run.smithi189.stdout:Install 7 Packages 2024-09-13T14:05:05.418 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:05.418 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 3.7 M 2024-09-13T14:05:05.418 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 80 M 2024-09-13T14:05:05.418 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:05.662 INFO:teuthology.orchestra.run.smithi189.stdout:(1/7): ceph-prometheus-alerts-17.2.7-1627.gbbda 78 kB/s | 15 kB 00:00 2024-09-13T14:05:05.679 INFO:teuthology.orchestra.run.smithi189.stdout:(2/7): ceph-grafana-dashboards-17.2.7-1627.gbbd 119 kB/s | 24 kB 00:00 2024-09-13T14:05:05.729 INFO:teuthology.orchestra.run.smithi189.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 135 kB/s | 9.0 kB 00:00 2024-09-13T14:05:05.754 INFO:teuthology.orchestra.run.smithi189.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 542 kB/s | 41 kB 00:00 2024-09-13T14:05:05.779 INFO:teuthology.orchestra.run.smithi189.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 611 kB/s | 31 kB 00:00 2024-09-13T14:05:05.813 INFO:teuthology.orchestra.run.smithi189.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-09-13T14:05:05.897 INFO:teuthology.orchestra.run.smithi189.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1627.gbbdadfa3 7.9 MB/s | 3.4 MB 00:00 2024-09-13T14:05:05.900 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:05.901 INFO:teuthology.orchestra.run.smithi189.stdout:Total 7.6 MB/s | 3.7 MB 00:00 2024-09-13T14:05:05.901 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:05.909 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:05.909 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:05.939 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:57 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:06.002 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:06.002 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:06.033 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-mgr-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:06.077 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:06.078 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:06.078 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:06.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:05:06.302 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:06.373 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-13T14:05:06.398 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-13T14:05:06.445 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-13T14:05:06.485 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-13T14:05:06.514 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 5/7 2024-09-13T14:05:06.726 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:58 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:06.863 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 3.4 M 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 24 k 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 15 k 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:05:06.864 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout:Installing weak dependencies: 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout:Install 7 Packages 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:06.865 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 3.7 M 2024-09-13T14:05:06.866 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 80 M 2024-09-13T14:05:06.866 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:07.350 INFO:teuthology.orchestra.run.smithi017.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1627.gbbd 161 kB/s | 24 kB 00:00 2024-09-13T14:05:07.375 INFO:teuthology.orchestra.run.smithi017.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1627.gbbda 83 kB/s | 15 kB 00:00 2024-09-13T14:05:07.409 INFO:teuthology.orchestra.run.smithi017.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 154 kB/s | 9.0 kB 00:00 2024-09-13T14:05:07.434 INFO:teuthology.orchestra.run.smithi017.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 1.3 MB/s | 31 kB 00:00 2024-09-13T14:05:07.451 INFO:teuthology.orchestra.run.smithi017.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 541 kB/s | 41 kB 00:00 2024-09-13T14:05:07.476 INFO:teuthology.orchestra.run.smithi017.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 4.4 MB/s | 188 kB 00:00 2024-09-13T14:05:07.510 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 6/7 2024-09-13T14:05:07.543 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:05:07.585 INFO:teuthology.orchestra.run.smithi017.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1627.gbbdadfa3 8.7 MB/s | 3.4 MB 00:00 2024-09-13T14:05:07.589 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:07.589 INFO:teuthology.orchestra.run.smithi017.stdout:Total 5.0 MB/s | 3.7 MB 00:00 2024-09-13T14:05:07.589 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:07.598 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:07.599 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:07.686 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:07.686 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:08.006 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-13T14:05:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-13T14:05:08.160 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-13T14:05:08.190 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-13T14:05:08.219 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 5/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 1/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 2/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 3/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-13T14:05:08.559 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-13T14:05:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:05:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:05:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:05:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:05:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:08.831 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:09.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:05:09.272 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 6/7 2024-09-13T14:05:09.314 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:05:09.712 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:01 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:09.844 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:09.845 INFO:teuthology.orchestra.run.smithi189.stdout:======================================================================================= 2024-09-13T14:05:09.845 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:05:09.845 INFO:teuthology.orchestra.run.smithi189.stdout:======================================================================================= 2024-09-13T14:05:09.845 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:09.845 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:05:09.846 INFO:teuthology.orchestra.run.smithi189.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout:======================================================================================= 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout:Install 12 Packages 2024-09-13T14:05:09.847 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:09.848 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 38 M 2024-09-13T14:05:09.848 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 212 M 2024-09-13T14:05:09.848 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:10.104 INFO:teuthology.orchestra.run.smithi189.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 921 kB/s | 191 kB 00:00 2024-09-13T14:05:10.246 INFO:teuthology.orchestra.run.smithi189.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.2 MB/s | 801 kB 00:00 2024-09-13T14:05:10.330 INFO:teuthology.orchestra.run.smithi189.stdout:(3/12): ceph-mgr-diskprediction-local-17.2.7-16 17 MB/s | 7.4 MB 00:00 2024-09-13T14:05:10.388 INFO:teuthology.orchestra.run.smithi189.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 115 kB/s | 33 kB 00:00 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 7/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9. 1/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarc 2/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.n 3/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-13T14:05:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-13T14:05:10.597 INFO:teuthology.orchestra.run.smithi189.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 185 kB/s | 38 kB 00:00 2024-09-13T14:05:10.622 INFO:teuthology.orchestra.run.smithi189.stdout:(6/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 62 kB/s | 18 kB 00:00 2024-09-13T14:05:10.671 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-13T14:05:10.671 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:05:10.672 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:10.673 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:10.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:05:11.089 INFO:teuthology.orchestra.run.smithi189.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 524 kB/s | 245 kB 00:00 2024-09-13T14:05:11.364 INFO:teuthology.orchestra.run.smithi189.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.7 MB/s | 3.0 MB 00:01 2024-09-13T14:05:11.475 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:03 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:11.489 INFO:teuthology.orchestra.run.smithi189.stdout:(9/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 5.5 MB/s | 4.9 MB 00:00 2024-09-13T14:05:11.515 INFO:teuthology.orchestra.run.smithi189.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.6 MB/s | 250 kB 00:00 2024-09-13T14:05:11.612 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 7.4 M 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:05:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:11.614 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-13T14:05:11.615 INFO:teuthology.orchestra.run.smithi017.stdout:Install 12 Packages 2024-09-13T14:05:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:11.615 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 38 M 2024-09-13T14:05:11.615 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 212 M 2024-09-13T14:05:11.615 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:11.868 INFO:teuthology.orchestra.run.smithi017.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.0 MB/s | 191 kB 00:00 2024-09-13T14:05:11.951 INFO:teuthology.orchestra.run.smithi017.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 392 kB/s | 33 kB 00:00 2024-09-13T14:05:11.976 INFO:teuthology.orchestra.run.smithi017.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.7 MB/s | 801 kB 00:00 2024-09-13T14:05:12.052 INFO:teuthology.orchestra.run.smithi017.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 242 kB/s | 18 kB 00:00 2024-09-13T14:05:12.144 INFO:teuthology.orchestra.run.smithi017.stdout:(5/12): ceph-mgr-diskprediction-local-17.2.7-16 16 MB/s | 7.4 MB 00:00 2024-09-13T14:05:12.169 INFO:teuthology.orchestra.run.smithi017.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 329 kB/s | 38 kB 00:00 2024-09-13T14:05:12.244 INFO:teuthology.orchestra.run.smithi017.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 10 MB/s | 3.0 MB 00:00 2024-09-13T14:05:12.303 INFO:teuthology.orchestra.run.smithi017.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.8 MB/s | 245 kB 00:00 2024-09-13T14:05:12.387 INFO:teuthology.orchestra.run.smithi017.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 36 MB/s | 5.1 MB 00:00 2024-09-13T14:05:12.428 INFO:teuthology.orchestra.run.smithi017.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.0 MB/s | 250 kB 00:00 2024-09-13T14:05:12.529 INFO:teuthology.orchestra.run.smithi017.stdout:(11/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 13 MB/s | 4.9 MB 00:00 2024-09-13T14:05:12.729 INFO:teuthology.orchestra.run.smithi017.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 48 MB/s | 16 MB 00:00 2024-09-13T14:05:12.730 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:12.730 INFO:teuthology.orchestra.run.smithi017.stdout:Total 34 MB/s | 38 MB 00:01 2024-09-13T14:05:12.998 INFO:teuthology.orchestra.run.smithi189.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.7 MB/s | 5.1 MB 00:01 2024-09-13T14:05:13.094 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:13.121 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:13.121 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:13.665 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:13.665 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:14.450 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:14.518 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:05:14.570 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:05:14.673 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:05:14.712 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:05:14.904 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:05:14.944 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:05:15.033 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:05:15.728 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:05:15.854 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:05:17.200 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:05:18.285 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:05:18.310 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:05:20.834 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:05:20.834 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 1/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:05:20.835 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:05:21.278 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:21.279 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:21.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:05:22.227 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:14 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:22.368 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:05:22.369 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout:Install 13 Packages 2024-09-13T14:05:22.370 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:22.371 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 2.2 M 2024-09-13T14:05:22.371 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 26 M 2024-09-13T14:05:22.371 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:22.837 INFO:teuthology.orchestra.run.smithi017.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 323 kB/s | 26 kB 00:00 2024-09-13T14:05:22.879 INFO:teuthology.orchestra.run.smithi017.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 158 kB/s | 19 kB 00:00 2024-09-13T14:05:22.954 INFO:teuthology.orchestra.run.smithi017.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.9 MB/s | 222 kB 00:00 2024-09-13T14:05:22.979 INFO:teuthology.orchestra.run.smithi017.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.6 MB/s | 159 kB 00:00 2024-09-13T14:05:23.004 INFO:teuthology.orchestra.run.smithi017.stdout:(5/13): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.el9 192 kB/s | 47 kB 00:00 2024-09-13T14:05:23.029 INFO:teuthology.orchestra.run.smithi017.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-13T14:05:23.055 INFO:teuthology.orchestra.run.smithi017.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.7 MB/s | 279 kB 00:00 2024-09-13T14:05:23.097 INFO:teuthology.orchestra.run.smithi017.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 350 kB/s | 32 kB 00:00 2024-09-13T14:05:23.114 INFO:teuthology.orchestra.run.smithi017.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 168 kB/s | 14 kB 00:00 2024-09-13T14:05:23.155 INFO:teuthology.orchestra.run.smithi017.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.5 MB/s | 220 kB 00:00 2024-09-13T14:05:23.172 INFO:teuthology.orchestra.run.smithi017.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.0 MB/s | 59 kB 00:00 2024-09-13T14:05:23.198 INFO:teuthology.orchestra.run.smithi017.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 2.1 MB/s | 90 kB 00:00 2024-09-13T14:05:23.248 INFO:teuthology.orchestra.run.smithi017.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.9 MB/s | 1.0 MB 00:00 2024-09-13T14:05:23.251 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:23.251 INFO:teuthology.orchestra.run.smithi017.stdout:Total 2.5 MB/s | 2.2 MB 00:00 2024-09-13T14:05:23.322 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:23.338 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:23.339 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:23.435 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:23.435 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:23.953 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:24.129 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:05:24.192 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:05:24.249 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:05:24.284 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:05:24.327 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:05:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:05:24.543 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:05:24.604 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:05:25.367 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:05:25.417 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:05:25.451 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:05:25.497 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:05:25.520 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:05:26.368 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:05:26.369 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:05:26.704 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:05:26.704 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:26.704 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:26.704 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:26.704 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:26.705 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:26.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:05:27.562 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:19 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:27.702 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:27.703 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:27.703 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:27.703 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:05:27.704 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:05:27.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:05:27.705 INFO:teuthology.orchestra.run.smithi017.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:05:27.705 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:27.705 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:27.705 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:27.706 INFO:teuthology.orchestra.run.smithi017.stdout:Install 6 Packages 2024-09-13T14:05:27.706 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:27.707 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 7.0 M 2024-09-13T14:05:27.707 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 33 M 2024-09-13T14:05:27.707 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:27.888 INFO:teuthology.orchestra.run.smithi189.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 1.0 MB/s | 16 MB 00:16 2024-09-13T14:05:27.889 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:27.889 INFO:teuthology.orchestra.run.smithi189.stdout:Total 2.1 MB/s | 38 MB 00:18 2024-09-13T14:05:28.051 INFO:teuthology.orchestra.run.smithi017.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.7 MB/s | 248 kB 00:00 2024-09-13T14:05:28.110 INFO:teuthology.orchestra.run.smithi017.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa3.e 611 kB/s | 124 kB 00:00 2024-09-13T14:05:28.151 INFO:teuthology.orchestra.run.smithi017.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.4 MB/s | 58 kB 00:00 2024-09-13T14:05:28.185 INFO:teuthology.orchestra.run.smithi017.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.0 MB/s | 548 kB 00:00 2024-09-13T14:05:28.210 INFO:teuthology.orchestra.run.smithi017.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.2 MB/s | 74 kB 00:00 2024-09-13T14:05:28.248 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:28.275 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:28.276 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:28.319 INFO:teuthology.orchestra.run.smithi017.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-09-13T14:05:28.321 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:28.321 INFO:teuthology.orchestra.run.smithi017.stdout:Total 11 MB/s | 7.0 MB 00:00 2024-09-13T14:05:28.395 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:28.407 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:28.407 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:28.512 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:28.514 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:28.809 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:28.809 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:28.829 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:29.012 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:05:29.080 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:05:29.565 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:29.633 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:05:29.646 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:05:29.678 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:05:29.737 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:05:29.786 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:05:29.823 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:05:29.829 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:05:29.858 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:05:30.016 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:05:30.045 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:05:30.130 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:05:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:05:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/6 2024-09-13T14:05:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:05:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:05:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:05:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:05:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:05:30.901 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:05:30.901 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:30.902 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:30.916 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:05:31.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-fuse 2024-09-13T14:05:31.719 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:23 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:31.858 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:31.859 INFO:teuthology.orchestra.run.smithi017.stdout:Install 2 Packages 2024-09-13T14:05:31.860 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:31.860 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 893 k 2024-09-13T14:05:31.860 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 2.6 M 2024-09-13T14:05:31.860 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:32.181 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:05:32.296 INFO:teuthology.orchestra.run.smithi017.stdout:(1/2): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-09-13T14:05:32.688 INFO:teuthology.orchestra.run.smithi017.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 114 kB/s | 80 kB 00:00 2024-09-13T14:05:32.689 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:32.689 INFO:teuthology.orchestra.run.smithi017.stdout:Total 1.1 MB/s | 893 kB 00:00 2024-09-13T14:05:32.703 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:32.728 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:32.728 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:32.816 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:32.816 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:33.012 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:33.146 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:05:33.215 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:05:33.220 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:05:33.243 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:05:33.893 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:05:33.894 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/2 2024-09-13T14:05:34.184 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:05:34.184 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:34.184 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:34.184 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:05:34.184 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:34.185 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:34.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados-devel 2024-09-13T14:05:35.026 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:27 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:35.164 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:35.165 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:35.165 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-13T14:05:35.165 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:35.165 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 126 k 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 448 k 2024-09-13T14:05:35.166 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:35.368 INFO:teuthology.orchestra.run.smithi017.stdout:librados-devel-17.2.7-1627.gbbdadfa3.el9.x86_64 627 kB/s | 126 kB 00:00 2024-09-13T14:05:35.368 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:35.368 INFO:teuthology.orchestra.run.smithi017.stdout:Total 622 kB/s | 126 kB 00:00 2024-09-13T14:05:35.369 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:35.374 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:35.374 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:35.443 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:35.443 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:35.603 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:35.710 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:35.869 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 12/12 2024-09-13T14:05:35.869 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdad 1/12 2024-09-13T14:05:35.869 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:05:35.870 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:05:35.871 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:05:35.871 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:05:36.167 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:36.293 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:05:36.293 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:05:36.294 INFO:teuthology.orchestra.run.smithi189.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:05:36.295 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:36.295 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:36.425 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:36.425 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:36.425 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:36.425 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:36.425 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:36.426 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:36.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs2 2024-09-13T14:05:36.652 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:05:37.237 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:29 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:37.239 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:29 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:37.333 INFO:teuthology.orchestra.run.smithi017.stdout:Package libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:37.370 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-rook noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 47 k 2024-09-13T14:05:37.371 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:37.372 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout:Install 13 Packages 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 2.2 M 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 26 M 2024-09-13T14:05:37.373 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:37.379 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:37.380 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:37.380 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:37.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs-devel 2024-09-13T14:05:37.867 INFO:teuthology.orchestra.run.smithi189.stdout:(1/13): ceph-mgr-rook-17.2.7-1627.gbbdadfa3.el9 297 kB/s | 47 kB 00:00 2024-09-13T14:05:37.993 INFO:teuthology.orchestra.run.smithi189.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 90 kB/s | 26 kB 00:00 2024-09-13T14:05:38.018 INFO:teuthology.orchestra.run.smithi189.stdout:(3/13): python3-jsonpointer-2.0-4.el9.noarch.rp 62 kB/s | 19 kB 00:00 2024-09-13T14:05:38.036 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:30 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:38.175 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:38.176 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:38.177 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:38.177 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 25 k 2024-09-13T14:05:38.177 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 110 k 2024-09-13T14:05:38.177 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:38.338 INFO:teuthology.orchestra.run.smithi017.stdout:libcephfs-devel-17.2.7-1627.gbbdadfa3.el9.x86_6 155 kB/s | 25 kB 00:00 2024-09-13T14:05:38.339 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:38.339 INFO:teuthology.orchestra.run.smithi017.stdout:Total 153 kB/s | 25 kB 00:00 2024-09-13T14:05:38.339 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:38.342 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:38.342 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:38.344 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:38.344 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:38.368 INFO:teuthology.orchestra.run.smithi189.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 423 kB/s | 159 kB 00:00 2024-09-13T14:05:38.410 INFO:teuthology.orchestra.run.smithi189.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 409 kB/s | 222 kB 00:00 2024-09-13T14:05:38.443 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:38.452 INFO:teuthology.orchestra.run.smithi189.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 771 kB/s | 32 kB 00:00 2024-09-13T14:05:38.477 INFO:teuthology.orchestra.run.smithi189.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 569 kB/s | 14 kB 00:00 2024-09-13T14:05:38.494 INFO:teuthology.orchestra.run.smithi189.stdout:(8/13): python3-requests-oauthlib-1.3.0-12.el9. 426 kB/s | 54 kB 00:00 2024-09-13T14:05:38.519 INFO:teuthology.orchestra.run.smithi189.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 556 kB/s | 279 kB 00:00 2024-09-13T14:05:38.553 INFO:teuthology.orchestra.run.smithi189.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.8 MB/s | 220 kB 00:00 2024-09-13T14:05:38.571 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:38.586 INFO:teuthology.orchestra.run.smithi189.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.6 MB/s | 90 kB 00:00 2024-09-13T14:05:38.611 INFO:teuthology.orchestra.run.smithi189.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 695 kB/s | 59 kB 00:00 2024-09-13T14:05:38.653 INFO:teuthology.orchestra.run.smithi189.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.4 MB/s | 1.0 MB 00:00 2024-09-13T14:05:38.656 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:38.656 INFO:teuthology.orchestra.run.smithi189.stdout:Total 1.7 MB/s | 2.2 MB 00:01 2024-09-13T14:05:38.721 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:38.736 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:38.736 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:38.823 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:38.824 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:38.937 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:39.166 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:39.166 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:39.166 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:39.166 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:39.166 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:39.167 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:39.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados2 2024-09-13T14:05:39.516 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:05:39.580 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:05:39.636 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:05:39.672 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:05:39.714 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:05:39.819 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:05:39.925 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:05:39.960 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:31 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:39.982 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:05:40.053 INFO:teuthology.orchestra.run.smithi017.stdout:Package librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:40.099 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:40.100 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:40.100 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:40.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librbd1 2024-09-13T14:05:40.703 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:05:40.746 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:05:40.766 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:40.792 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:05:40.833 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:05:40.857 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:05:40.859 INFO:teuthology.orchestra.run.smithi017.stdout:Package librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:40.905 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:40.906 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:40.907 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:40.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rados 2024-09-13T14:05:41.574 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:33 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:41.590 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 13/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:05:41.591 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:05:41.667 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:41.713 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:41.714 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:41.714 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:41.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rgw 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-rook-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:05:41.917 INFO:teuthology.orchestra.run.smithi189.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:41.918 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:42.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:05:42.374 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:42.467 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:42.512 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:42.513 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:42.513 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:42.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-cephfs 2024-09-13T14:05:42.764 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:42.898 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:42.898 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1627.gbbdadfa3.el9 ceph-noarch 124 k 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:05:42.899 INFO:teuthology.orchestra.run.smithi189.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout:Install 6 Packages 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:42.900 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 7.0 M 2024-09-13T14:05:42.901 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 33 M 2024-09-13T14:05:42.901 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:43.171 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:35 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:43.188 INFO:teuthology.orchestra.run.smithi189.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.8 MB/s | 248 kB 00:00 2024-09-13T14:05:43.255 INFO:teuthology.orchestra.run.smithi189.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1627.gbbdadfa3.e 611 kB/s | 124 kB 00:00 2024-09-13T14:05:43.266 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:43.280 INFO:teuthology.orchestra.run.smithi189.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 5.8 MB/s | 548 kB 00:00 2024-09-13T14:05:43.311 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:43.313 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:43.313 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:43.314 INFO:teuthology.orchestra.run.smithi189.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.7 MB/s | 74 kB 00:00 2024-09-13T14:05:43.347 INFO:teuthology.orchestra.run.smithi189.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 626 kB/s | 58 kB 00:00 2024-09-13T14:05:43.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rbd 2024-09-13T14:05:43.431 INFO:teuthology.orchestra.run.smithi189.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 16 MB/s | 6.0 MB 00:00 2024-09-13T14:05:43.433 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:43.434 INFO:teuthology.orchestra.run.smithi189.stdout:Total 13 MB/s | 7.0 MB 00:00 2024-09-13T14:05:43.504 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:43.516 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:43.516 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:43.615 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:43.616 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:43.922 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:43.975 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:35 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:44.069 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:44.100 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:05:44.114 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:44.115 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-13T14:05:44.116 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:44.166 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:05:44.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-fuse 2024-09-13T14:05:44.709 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:05:44.779 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:36 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:44.804 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:05:44.866 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:05:44.911 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:05:44.918 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:44.918 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 82 k 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 222 k 2024-09-13T14:05:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:45.125 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 398 kB/s | 82 kB 00:00 2024-09-13T14:05:45.126 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:45.126 INFO:teuthology.orchestra.run.smithi017.stdout:Total 395 kB/s | 82 kB 00:00 2024-09-13T14:05:45.126 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:45.133 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:45.133 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:45.200 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:45.200 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:45.363 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:45.475 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:45.685 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 6/6 2024-09-13T14:05:45.686 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 1/6 2024-09-13T14:05:45.686 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:05:45.686 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:05:45.686 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:05:45.686 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:05:45.847 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:45.949 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:05:45.949 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-cephadm-2:17.2.7-1627.gbbdadfa3.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:45.950 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:46.102 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:46.102 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:46.103 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:46.103 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:46.103 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:46.103 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:46.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install ceph-fuse 2024-09-13T14:05:46.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-mirror 2024-09-13T14:05:46.797 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:38 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:46.920 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:38 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:46.930 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repository Size 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 813 k 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:46.931 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:46.932 INFO:teuthology.orchestra.run.smithi189.stdout:Install 2 Packages 2024-09-13T14:05:46.932 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:46.932 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 893 k 2024-09-13T14:05:46.932 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 2.6 M 2024-09-13T14:05:46.932 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:47.058 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:47.059 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:47.060 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:47.060 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:47.060 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 2.9 M 2024-09-13T14:05:47.060 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 12 M 2024-09-13T14:05:47.060 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:47.215 INFO:teuthology.orchestra.run.smithi189.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 678 kB/s | 80 kB 00:00 2024-09-13T14:05:47.399 INFO:teuthology.orchestra.run.smithi189.stdout:(2/2): ceph-fuse-17.2.7-1627.gbbdadfa3.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-09-13T14:05:47.399 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:47.400 INFO:teuthology.orchestra.run.smithi189.stdout:Total 1.9 MB/s | 893 kB 00:00 2024-09-13T14:05:47.411 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:47.433 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:47.433 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:47.495 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-mirror-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 6.7 MB/s | 2.9 MB 00:00 2024-09-13T14:05:47.496 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:47.497 INFO:teuthology.orchestra.run.smithi017.stdout:Total 6.7 MB/s | 2.9 MB 00:00 2024-09-13T14:05:47.497 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:47.506 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:47.506 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:47.513 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:47.513 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:47.591 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:47.591 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:47.695 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:47.837 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:05:47.914 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:05:48.020 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:48.088 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:48.113 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:48.114 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:05:48.114 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:05:48.114 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:05:48.114 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:05:48.114 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:48.554 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2/2 2024-09-13T14:05:48.555 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/2 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:48.848 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:49.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install librados-devel 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:49.200 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:49.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-nbd 2024-09-13T14:05:49.734 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:41 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:49.867 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:49.867 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:49.867 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repo Size 2024-09-13T14:05:49.867 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout: librados-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 126 k 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:05:49.868 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:49.869 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 126 k 2024-09-13T14:05:49.869 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 448 k 2024-09-13T14:05:49.869 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:50.055 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:42 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:50.077 INFO:teuthology.orchestra.run.smithi189.stdout:librados-devel-17.2.7-1627.gbbdadfa3.el9.x86_64 604 kB/s | 126 kB 00:00 2024-09-13T14:05:50.078 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:50.078 INFO:teuthology.orchestra.run.smithi189.stdout:Total 600 kB/s | 126 kB 00:00 2024-09-13T14:05:50.078 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:50.084 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:50.084 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:50.149 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:50.149 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:50.194 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-13T14:05:50.194 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:50.194 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-13T14:05:50.195 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:50.196 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 165 k 2024-09-13T14:05:50.196 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 477 k 2024-09-13T14:05:50.196 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-13T14:05:50.298 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:50.392 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:50.417 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 748 kB/s | 165 kB 00:00 2024-09-13T14:05:50.417 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:50.417 INFO:teuthology.orchestra.run.smithi017.stdout:Total 742 kB/s | 165 kB 00:00 2024-09-13T14:05:50.418 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-13T14:05:50.426 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-13T14:05:50.426 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-13T14:05:50.494 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-13T14:05:50.494 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-13T14:05:50.670 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-13T14:05:50.783 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:51.107 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:51.107 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:51.108 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:51.108 INFO:teuthology.orchestra.run.smithi189.stdout: librados-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:51.108 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:51.108 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:51.204 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:51.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install libcephfs2 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:05:51.463 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-13T14:05:51.721 DEBUG:teuthology.parallel:result is None 2024-09-13T14:05:51.966 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:43 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:52.057 INFO:teuthology.orchestra.run.smithi189.stdout:Package libcephfs2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:52.103 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:52.104 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:52.104 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:52.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install libcephfs-devel 2024-09-13T14:05:52.749 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:44 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:52.883 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:52.885 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:52.885 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repo Size 2024-09-13T14:05:52.885 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:52.885 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout: libcephfs-devel x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 25 k 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 25 k 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 110 k 2024-09-13T14:05:52.886 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:53.030 INFO:teuthology.orchestra.run.smithi189.stdout:libcephfs-devel-17.2.7-1627.gbbdadfa3.el9.x86_6 172 kB/s | 25 kB 00:00 2024-09-13T14:05:53.031 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:53.031 INFO:teuthology.orchestra.run.smithi189.stdout:Total 170 kB/s | 25 kB 00:00 2024-09-13T14:05:53.031 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:53.034 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:53.034 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:53.036 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:53.036 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:53.123 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:53.227 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:53.577 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout: libcephfs-devel-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:53.805 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:54.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install librados2 2024-09-13T14:05:54.582 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:46 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:54.673 INFO:teuthology.orchestra.run.smithi189.stdout:Package librados2-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:54.716 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:54.717 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:54.717 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:54.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install librbd1 2024-09-13T14:05:55.364 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:47 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:55.456 INFO:teuthology.orchestra.run.smithi189.stdout:Package librbd1-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:55.499 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:55.500 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:55.501 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:55.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install python3-rados 2024-09-13T14:05:56.149 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:48 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:56.240 INFO:teuthology.orchestra.run.smithi189.stdout:Package python3-rados-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:56.283 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:56.284 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:56.284 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:56.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install python3-rgw 2024-09-13T14:05:56.939 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:48 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:57.029 INFO:teuthology.orchestra.run.smithi189.stdout:Package python3-rgw-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:57.073 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:57.074 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:57.074 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:57.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install python3-cephfs 2024-09-13T14:05:57.719 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:49 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:57.810 INFO:teuthology.orchestra.run.smithi189.stdout:Package python3-cephfs-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:57.852 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:57.853 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:57.853 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:57.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install python3-rbd 2024-09-13T14:05:58.510 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:50 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:58.600 INFO:teuthology.orchestra.run.smithi189.stdout:Package python3-rbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 is already installed. 2024-09-13T14:05:58.643 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:58.644 INFO:teuthology.orchestra.run.smithi189.stdout:Nothing to do. 2024-09-13T14:05:58.644 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:05:58.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install rbd-fuse 2024-09-13T14:05:59.296 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:51 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:05:59.431 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-fuse x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 82 k 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:59.432 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 82 k 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 222 k 2024-09-13T14:05:59.433 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:05:59.625 INFO:teuthology.orchestra.run.smithi189.stdout:rbd-fuse-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 426 kB/s | 82 kB 00:00 2024-09-13T14:05:59.626 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:05:59.626 INFO:teuthology.orchestra.run.smithi189.stdout:Total 423 kB/s | 82 kB 00:00 2024-09-13T14:05:59.626 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:05:59.633 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:05:59.633 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:05:59.697 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:05:59.697 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:05:59.840 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:05:59.947 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:00.322 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:00.573 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:00.574 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:00.574 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:06:00.574 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-fuse-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:06:00.574 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:00.574 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:06:00.834 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install rbd-mirror 2024-09-13T14:06:01.412 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:53 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:06:01.550 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:06:01.550 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout: Package Arch Version Repo Size 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-mirror x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 2.9 M 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:06:01.551 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:01.552 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 2.9 M 2024-09-13T14:06:01.552 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 12 M 2024-09-13T14:06:01.552 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:06:01.903 INFO:teuthology.orchestra.run.smithi189.stdout:rbd-mirror-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 8.4 MB/s | 2.9 MB 00:00 2024-09-13T14:06:01.903 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:06:01.904 INFO:teuthology.orchestra.run.smithi189.stdout:Total 8.3 MB/s | 2.9 MB 00:00 2024-09-13T14:06:01.904 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:06:01.913 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:06:01.913 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:06:01.994 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:06:01.995 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:06:02.417 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:06:02.495 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:02.521 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:02.522 INFO:teuthology.orchestra.run.smithi189.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:06:02.522 INFO:teuthology.orchestra.run.smithi189.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:06:02.522 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:06:02.522 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:06:02.522 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:03.605 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:03.605 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:03.606 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:06:03.606 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-mirror-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:06:03.606 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:03.606 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:06:03.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum -y install rbd-nbd 2024-09-13T14:06:04.435 INFO:teuthology.orchestra.run.smithi189.stdout:Last metadata expiration check: 0:01:56 ago on Fri 13 Sep 2024 02:04:08 PM UTC. 2024-09-13T14:06:04.576 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-09-13T14:06:04.577 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:04.577 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-09-13T14:06:04.577 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-nbd x86_64 2:17.2.7-1627.gbbdadfa3.el9 ceph 165 k 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Install 1 Package 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 165 k 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 477 k 2024-09-13T14:06:04.578 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-09-13T14:06:04.789 INFO:teuthology.orchestra.run.smithi189.stdout:rbd-nbd-17.2.7-1627.gbbdadfa3.el9.x86_64.rpm 781 kB/s | 165 kB 00:00 2024-09-13T14:06:04.790 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:06:04.791 INFO:teuthology.orchestra.run.smithi189.stdout:Total 775 kB/s | 165 kB 00:00 2024-09-13T14:06:04.791 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-09-13T14:06:04.799 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-09-13T14:06:04.799 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-09-13T14:06:04.868 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-09-13T14:06:04.868 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-09-13T14:06:05.038 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-09-13T14:06:05.140 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 1/1 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout: rbd-nbd-2:17.2.7-1627.gbbdadfa3.el9.x86_64 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:06:05.863 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-09-13T14:06:06.096 DEBUG:teuthology.parallel:result is None 2024-09-13T14:06:06.096 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:06:06.096 INFO:teuthology.packaging:ref: None 2024-09-13T14:06:06.096 INFO:teuthology.packaging:tag: None 2024-09-13T14:06:06.096 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:06:06.096 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:06:06.097 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:06:06.324 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:06:06.351 INFO:teuthology.orchestra.run.smithi017.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:06:06.351 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:06:06.351 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:06:06.353 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:06:06.353 INFO:teuthology.packaging:ref: None 2024-09-13T14:06:06.353 INFO:teuthology.packaging:tag: None 2024-09-13T14:06:06.353 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:06:06.353 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:06:06.353 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-13T14:06:06.574 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:06:06.596 INFO:teuthology.orchestra.run.smithi189.stdout:17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:06:06.597 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1627.gbbdadfa3.el9 2024-09-13T14:06:06.597 INFO:teuthology.task.install:The correct ceph version 17.2.7-1627.gbbdadfa3 is installed. 2024-09-13T14:06:06.599 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T14:06:06.599 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:06:06.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:06:06.632 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:06:06.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:06:06.663 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T14:06:06.663 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:06:06.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:06:06.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:06:06.769 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:06:06.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:06:06.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:06:06.879 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T14:06:06.879 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:06:06.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:06:06.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:06:06.975 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:06:06.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:06:07.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:06:07.065 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T14:06:07.066 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:06:07.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:06:07.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:06:07.161 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:06:07.162 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:06:07.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:06:07.252 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T14:06:07.349 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:06:07.349 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:06:07.349 INFO:tasks.cephadm:Cluster fsid is 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:06:07.350 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T14:06:07.350 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.17', 'mon.c': '[v2:172.21.15.17:3301,v1:172.21.15.17:6790]', 'mon.b': '172.21.15.189'} 2024-09-13T14:06:07.350 INFO:tasks.cephadm:First mon is mon.a on smithi017 2024-09-13T14:06:07.350 INFO:tasks.cephadm:First mgr is y 2024-09-13T14:06:07.350 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T14:06:07.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo hostname $(hostname -s) 2024-09-13T14:06:07.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo hostname $(hostname -s) 2024-09-13T14:06:07.404 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-13T14:06:07.404 DEBUG:teuthology.orchestra.run.smithi017:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.642 INFO:teuthology.orchestra.run.smithi017.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:06 /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.642 DEBUG:teuthology.orchestra.run.smithi189:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.779 INFO:teuthology.orchestra.run.smithi189.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 13 14:06 /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.779 DEBUG:teuthology.orchestra.run.smithi017:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.799 DEBUG:teuthology.orchestra.run.smithi189:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:06:07.825 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-13T14:06:07.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:06:07.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:06:09.388 INFO:teuthology.orchestra.run.smithi017.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:06:09.519 INFO:teuthology.orchestra.run.smithi189.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout: "repo_digests": [ 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:06:43.725 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout: "repo_digests": [ 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:06:47.439 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:06:47.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo mkdir -p /etc/ceph 2024-09-13T14:06:47.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo mkdir -p /etc/ceph 2024-09-13T14:06:47.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 777 /etc/ceph 2024-09-13T14:06:47.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 777 /etc/ceph 2024-09-13T14:06:47.586 INFO:tasks.cephadm:Writing seed config... 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T14:06:47.588 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-13T14:06:47.589 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T14:06:47.589 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T14:06:47.589 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T14:06:47.589 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T14:06:47.589 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T14:06:47.589 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:06:47.590 DEBUG:teuthology.orchestra.run.smithi017:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T14:06:47.619 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 523f52e4-71d9-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T14:06:47.620 DEBUG:teuthology.orchestra.run.smithi017:mon.a> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T14:06:47.664 DEBUG:teuthology.orchestra.run.smithi017:mgr.y> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T14:06:47.708 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T14:06:47.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.17 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:06:47.919 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:06:47.920 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '523f52e4-71d9-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.17', '--skip-admin-label'] 2024-09-13T14:06:47.945 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:06:47.945 INFO:teuthology.orchestra.run.smithi017.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T14:06:47.945 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying podman|docker is present... 2024-09-13T14:06:47.968 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:06:47.968 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying lvm2 is present... 2024-09-13T14:06:47.968 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying time synchronization is in place... 2024-09-13T14:06:47.976 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:06:47.976 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:06:47.983 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:06:47.983 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:06:47.991 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2024-09-13T14:06:47.997 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2024-09-13T14:06:47.998 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:06:47.998 INFO:teuthology.orchestra.run.smithi017.stdout:Repeating the final host check... 2024-09-13T14:06:48.020 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:06:48.020 INFO:teuthology.orchestra.run.smithi017.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-13T14:06:48.021 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl is present 2024-09-13T14:06:48.021 INFO:teuthology.orchestra.run.smithi017.stdout:lvcreate is present 2024-09-13T14:06:48.028 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:06:48.028 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:06:48.034 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:06:48.035 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:06:48.042 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:Host looks OK 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:Cluster fsid: 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:Acquiring lock 139796830770080 on /run/cephadm/523f52e4-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:06:48.049 INFO:teuthology.orchestra.run.smithi017.stdout:Lock 139796830770080 acquired on /run/cephadm/523f52e4-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:06:48.050 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 3300 ... 2024-09-13T14:06:48.050 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 6789 ... 2024-09-13T14:06:48.050 INFO:teuthology.orchestra.run.smithi017.stdout:Base mon IP(s) is [172.21.15.17:3300, 172.21.15.17:6789], mon addrv is [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2024-09-13T14:06:48.054 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.17 metric 100 2024-09-13T14:06:48.054 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.17 metric 100 2024-09-13T14:06:48.057 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T14:06:48.057 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T14:06:48.057 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1793sec hoplimit 64 pref medium 2024-09-13T14:06:48.059 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T14:06:48.059 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T14:06:48.060 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:06:48.060 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T14:06:48.060 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe59:d992/64 scope link 2024-09-13T14:06:48.060 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:06:48.060 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2024-09-13T14:06:48.061 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2024-09-13T14:06:48.061 INFO:teuthology.orchestra.run.smithi017.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-13T14:06:48.061 INFO:teuthology.orchestra.run.smithi017.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T14:06:48.061 INFO:teuthology.orchestra.run.smithi017.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:06:48.532 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:06:48.532 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:06:48.532 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Getting image source signatures 2024-09-13T14:06:48.532 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-13T14:06:48.533 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:ca308b3f556da579f99cd79db684ebbf678db4c29a649b66aaf6f99bc8124cd6 2024-09-13T14:06:48.533 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying config sha256:a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c 2024-09-13T14:06:48.533 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-13T14:06:49.520 INFO:teuthology.orchestra.run.smithi017.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:06:49.521 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:06:49.521 INFO:teuthology.orchestra.run.smithi017.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T14:06:50.442 INFO:teuthology.orchestra.run.smithi017.stdout:stat: stdout 167 167 2024-09-13T14:06:50.442 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial keys... 2024-09-13T14:06:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQD6RuRmNZIlMRAARr6urod86K96Pdr5SGwhwg== 2024-09-13T14:06:52.396 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQD7RuRmQZQWLhAAJHcxEo6XlQjgMhnT7GWaag== 2024-09-13T14:06:53.418 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQD8RuRmWOf1LRAAQ74rY4qfHSFyopb1FD2uqw== 2024-09-13T14:06:53.418 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial monmap... 2024-09-13T14:06:54.440 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:monmaptool for a [v2:172.21.15.17:3300,v1:172.21.15.17:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:setting min_mon_release = octopus 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: set fsid to 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:06:54.441 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mon... 2024-09-13T14:06:55.404 INFO:teuthology.orchestra.run.smithi017.stdout:create mon.a on 2024-09-13T14:06:55.613 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-13T14:06:55.763 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T14:06:55.913 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:06:55.914 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:06:56.090 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:06:56.091 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T14:06:56.261 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target.wants/ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:06:56.747 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:56 smithi017 ceph-mon[28924]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:06:56.747 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:56 smithi017 bash[28886]: 4ca2d2593414022cef329ddf8897c7f66dea44d3b9c2c1f581a6467efb03775a 2024-09-13T14:06:56.761 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:06:56.761 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon to start... 2024-09-13T14:06:56.769 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon... 2024-09-13T14:06:57.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:56 smithi017 systemd[1]: Started Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:06:57.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:06:57.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:06:57.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: fsmap 2024-09-13T14:06:57.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:06:57.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: mgrmap e1: no daemons active 2024-09-13T14:06:57.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:57 smithi017 ceph-mon[28924]: from='client.? 172.21.15.17:0/842307551' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T14:06:57.814 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T14:06:57.815 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout id: 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout services: 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.560492s) 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout data: 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T14:06:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:06:57.817 INFO:teuthology.orchestra.run.smithi017.stdout:mon is available 2024-09-13T14:06:57.817 INFO:teuthology.orchestra.run.smithi017.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T14:06:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:58 smithi017 ceph-mon[28924]: from='client.? 172.21.15.17:0/2835525205' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:06:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:06:58 smithi017 ceph-mon[28924]: from='client.? 172.21.15.17:0/2835525205' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T14:07:03.355 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:03.355 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:07:03.355 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:07:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:07:03.357 INFO:teuthology.orchestra.run.smithi017.stdout:Generating new minimal ceph.conf... 2024-09-13T14:07:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:05 smithi017 ceph-mon[28924]: from='client.? 172.21.15.17:0/4037246855' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:07:06.183 INFO:teuthology.orchestra.run.smithi017.stdout:Restarting the monitor... 2024-09-13T14:07:06.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 systemd[1]: Stopping Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:07:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[28901]: 2024-09-13T14:07:06.443+0000 7f08b9e54640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:07:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[28901]: 2024-09-13T14:07:06.443+0000 7f08b9e54640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T14:07:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 podman[29244]: 2024-09-13 14:07:06.454008716 +0000 UTC m=+0.106934440 container died 4ca2d2593414022cef329ddf8897c7f66dea44d3b9c2c1f581a6467efb03775a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T14:07:07.008 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 podman[29244]: 2024-09-13 14:07:06.791044602 +0000 UTC m=+0.443970367 container cleanup 4ca2d2593414022cef329ddf8897c7f66dea44d3b9c2c1f581a6467efb03775a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T14:07:07.009 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:06 smithi017 bash[29244]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:07:07.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 podman[29261]: 2024-09-13 14:07:07.008456759 +0000 UTC m=+0.550306288 container remove 4ca2d2593414022cef329ddf8897c7f66dea44d3b9c2c1f581a6467efb03775a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T14:07:07.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:07:07.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 systemd[1]: Stopped Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:07:07.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 systemd[1]: Starting Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:07:07.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 podman[29360]: 2024-09-13 14:07:07.468027633 +0000 UTC m=+0.111549497 container create 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T14:07:07.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 podman[29360]: 2024-09-13 14:07:07.372472794 +0000 UTC m=+0.015994672 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:07:07.774 INFO:teuthology.orchestra.run.smithi017.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-13T14:07:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 podman[29360]: 2024-09-13 14:07:07.678330094 +0000 UTC m=+0.321851959 container init 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 podman[29360]: 2024-09-13 14:07:07.683375015 +0000 UTC m=+0.326896884 container start 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: pidfile_write: ignore empty --pid-file 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: load: jerasure load: lrc 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:07:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: DB SUMMARY 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: DB Session ID: G999OSD6142QSFW6Z37M 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83684 ; 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.env: 0x55b97d535440 2024-09-13T14:07:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.fs: Posix File System 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.info_log: 0x55b97fa94ec0 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.statistics: (nil) 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.use_fsync: 0 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:07:07.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:07:07.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:07:07.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.db_log_dir: 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:07:07.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.write_buffer_manager: 0x55b97fb527e0 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:07:07.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.unordered_write: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.row_cache: None 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.wal_filter: None 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:07:07.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:07:07.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_open_files: -1 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:07:07.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Compression algorithms supported: 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kZSTD supported: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:07:07.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.merge_operator: 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_filter: None 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b97fa3ba38) 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:07:07.961 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: hash_index_allow_collision: 1 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 1 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x55b97fa711f0 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2024-09-13T14:07:07.962 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:07:07.963 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 4 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression: NoCompression 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.num_levels: 7 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:07:07.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:07:07.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:07:07.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:07:07.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:07:07.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.ttl: 2592000 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:07:07.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236427708115, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236427708902, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80257, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78499, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72712, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236427, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "48c2b631-e091-4dda-a6ae-b1bde03a85b8", "db_session_id": "G999OSD6142QSFW6Z37M"}} 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236427709264, "job": 1, "event": "recovery_finished"} 2024-09-13T14:07:07.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55b97faa4700 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: DB pointer 0x55b97fa74000 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:07:07.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:07:07.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 23.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 23.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:07:07.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.2 0.00 0.00 1 0.001 0 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:07:07.974 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 23.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???) e1 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).mds e1 new map 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).mds e1 print_map 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: e1 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: -1 2024-09-13T14:07:07.975 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout: No filesystems configured 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: fsmap 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 ceph-mon[29399]: mgrmap e1: no daemons active 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 bash[29360]: 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 2024-09-13T14:07:07.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:07 smithi017 systemd[1]: Started Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:07:08.812 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T14:07:08.812 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:07:08.813 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mgr... 2024-09-13T14:07:08.813 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T14:07:08.813 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T14:07:09.001 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:07:09.001 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T14:07:09.154 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968.target.wants/ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-523f52e4-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:07:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:09 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3947896717' entity='client.admin' 2024-09-13T14:07:09.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 podman[29635]: 2024-09-13 14:07:09.28665217 +0000 UTC m=+0.016265805 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:07:09.702 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:07:09.702 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-13T14:07:09.710 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:07:09.710 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:07:09.710 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-13T14:07:09.710 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:07:09.719 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:07:09.720 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr to start... 2024-09-13T14:07:09.728 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr... 2024-09-13T14:07:09.832 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 podman[29635]: 2024-09-13 14:07:09.579337629 +0000 UTC m=+0.308951268 container init 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:07:09.832 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 podman[29635]: 2024-09-13 14:07:09.584496231 +0000 UTC m=+0.314109858 container start 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:07:09.832 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 bash[29635]: 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 2024-09-13T14:07:09.832 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 systemd[1]: Started Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:07:09.832 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:09.702+0000 7faa7e1baec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:07:09.833 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:09.763+0000 7faa7e1baec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:09.833 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:09.831+0000 7faa7e1baec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:07:10.110 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:09 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:09.962+0000 7faa7e1baec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:10.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:10 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3173579661' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:07:10.387 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:10 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:10.108+0000 7faa7e1baec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:07:10.652 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:10 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:10.386+0000 7faa7e1baec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:07:10.652 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:10 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:10.449+0000 7faa7e1baec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:07:10.652 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:10 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:10.620+0000 7faa7e1baec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:07:11.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:11 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:11.508+0000 7faa7e1baec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:07:11.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:11 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:11.576+0000 7faa7e1baec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:07:12.636 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.329+0000 7faa7e1baec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:07:12.636 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.445+0000 7faa7e1baec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:07:12.636 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.573+0000 7faa7e1baec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:12.636 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.635+0000 7faa7e1baec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:07:12.899 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.767+0000 7faa7e1baec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:07:12.899 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.837+0000 7faa7e1baec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:07:12.899 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:12.898+0000 7faa7e1baec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:07:13.175 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.039+0000 7faa7e1baec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:07:13.175 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.105+0000 7faa7e1baec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:07:13.175 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.174+0000 7faa7e1baec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:07:13.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.259+0000 7faa7e1baec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:13.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.319+0000 7faa7e1baec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:07:13.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:13.378+0000 7faa7e1baec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:07:13.566 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:13.566 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:13.566 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "523f52e4-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:07:13.566 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:07:13.567 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.568 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:07:13.570 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:06:56.702185+0000", 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:13.571 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (1/15)... 2024-09-13T14:07:14.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:14.167+0000 7faa7e1baec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:07:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: Activating manager daemon y 2024-09-13T14:07:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: mgrmap e2: y(active, starting, since 0.0028886s) 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: Manager daemon y is now available 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' 2024-09-13T14:07:14.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:14 smithi017 ceph-mon[29399]: from='mgr.14100 172.21.15.17:0/3387906660' entity='mgr.y' 2024-09-13T14:07:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:16 smithi017 ceph-mon[29399]: mgrmap e3: y(active, since 1.00725s) 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "523f52e4-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.799 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.800 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:07:16.801 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:07:16.804 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:07:16.805 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:06:56.702185+0000", 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:07:16.806 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:07:16.807 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:16.807 INFO:teuthology.orchestra.run.smithi017.stdout:mgr is available 2024-09-13T14:07:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:17 smithi017 ceph-mon[29399]: mgrmap e4: y(active, since 2s) 2024-09-13T14:07:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:17 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/772257559' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:07:17.945 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:07:17.946 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:07:17.947 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling cephadm module... 2024-09-13T14:07:18.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:18 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2427632206' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:07:19.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4176714004' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T14:07:19.449 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setuser ceph since I am not root 2024-09-13T14:07:19.449 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setgroup ceph since I am not root 2024-09-13T14:07:19.449 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.321+0000 7f6319a46ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:07:19.449 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.377+0000 7f6319a46ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:19.449 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.446+0000 7f6319a46ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:07:19.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.575+0000 7f6319a46ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:19.989 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.718+0000 7f6319a46ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:07:19.990 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:19.987+0000 7f6319a46ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:07:20.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:20 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4176714004' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T14:07:20.252 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:20 smithi017 ceph-mon[29399]: mgrmap e5: y(active, since 5s) 2024-09-13T14:07:20.252 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:20 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:20.053+0000 7f6319a46ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:07:20.252 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:20 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:20.225+0000 7f6319a46ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:07:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:21 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1125648792' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:07:21.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:21.108+0000 7f6319a46ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:07:21.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:21.175+0000 7f6319a46ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2024-09-13T14:07:21.551 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 5... 2024-09-13T14:07:22.045 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:21.926+0000 7f6319a46ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:07:22.046 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.043+0000 7f6319a46ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:07:22.363 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.168+0000 7f6319a46ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:22.363 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.230+0000 7f6319a46ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:07:22.363 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.360+0000 7f6319a46ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:07:22.632 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.430+0000 7f6319a46ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:07:22.632 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.490+0000 7f6319a46ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:07:22.632 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.629+0000 7f6319a46ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:07:22.913 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.695+0000 7f6319a46ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:07:22.913 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.763+0000 7f6319a46ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:07:22.913 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.850+0000 7f6319a46ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:22.913 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.911+0000 7f6319a46ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:07:23.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:22.971+0000 7f6319a46ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:07:24.077 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:23.762+0000 7f6319a46ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: Active manager daemon y restarted 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: Activating manager daemon y 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: mgrmap e6: y(active, starting, since 0.0032157s) 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: Manager daemon y is now available 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:24.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:24.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:24.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:24.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:07:24.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:23 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:07:25.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:24 smithi017 ceph-mon[29399]: Found migration_current of "None". Setting to last migration. 2024-09-13T14:07:25.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:24 smithi017 ceph-mon[29399]: mgrmap e7: y(active, since 1.00536s) 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 5 is available 2024-09-13T14:07:25.356 INFO:teuthology.orchestra.run.smithi017.stdout:Setting orchestrator backend to cephadm... 2024-09-13T14:07:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:25 smithi017 ceph-mon[29399]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:07:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:25 smithi017 ceph-mon[29399]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:07:27.050 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:26 smithi017 ceph-mon[29399]: mgrmap e8: y(active, since 2s) 2024-09-13T14:07:27.050 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:26 smithi017 ceph-mon[29399]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:27.050 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:26 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:27.050 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:26 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:27.617 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T14:07:27.617 INFO:teuthology.orchestra.run.smithi017.stdout:Generating ssh key... 2024-09-13T14:07:27.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:27 smithi017 ceph-mon[29399]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Generating public/private rsa key pair. 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Your identification has been saved in /tmp/tmpnxiu9cv4/key 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Your public key has been saved in /tmp/tmpnxiu9cv4/key.pub 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: The key fingerprint is: 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: SHA256:Ui2ud7CASjpkOUdvrD0CKz0SyAHTcO2M3e7Pfp3wWGc ceph-523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: The key's randomart image is: 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: +---[RSA 3072]----+ 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |+o.. | 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |.o. . . | 2024-09-13T14:07:28.697 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: | . * . o . | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |o = *..o . | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |oB...=o S | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |o==.+ .+ o. . E | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |=.+o +. o .* + | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: |.o .. oo .o + | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: | o+. | 2024-09-13T14:07:28.698 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:28 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: +----[SHA256]-----+ 2024-09-13T14:07:29.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-mon[29399]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:29.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-mon[29399]: Generating ssh key... 2024-09-13T14:07:29.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:29.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:29.720+0000 7f62cbdf5640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:07:29] ENGINE Error in HTTPServer.serve 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Traceback (most recent call last): 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._connections.run(self.expiration_interval) 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._run(expiration_interval) 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:07:29.920 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s = self.context.wrap_socket( 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: return self.sslsocket_class._create( 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self.do_handshake() 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._sslobj.do_handshake() 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:07:29.921 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:30.180 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKY7NoEyCCkyxmlFcbbbJsX+bBik7S//n6sd3aNF0+5lWuYd0TBob2Su58Iw0mKFbRpHKs8M6nhmFqTNqSa4+tlGuU4ybJMrMn/jWzFaR7ktxFeDglkNWXj8yxUMfif9gB3zgujIT+NT3yM9SifswNX55KeA8vxDOIrfzY7AD3S0zRwjjDkxXIE9NB5V/OohYF6YdNTFgsdS6zdIwgRIXLyQGXK9CdjRekBYM69yFQCYiCEbKoCvSp1HAemHK2tbMAkHNk43qTsBZYFAsbaG16v3kFkBGRb59BoHRQjIY4VW1tgCWkUE9tXHWHPp+2CtBsTQ1ODHNu2qaY1Ayw+A49Fxm03LnxphBQxIgrA71LT/HMIHSXe76SwXkSHCEge81KO/1nSIbEkQbpTKQJE8MoTw/XEq0kKo1K9cJxjxN9o0g2+ARxe5sMLl/AwH/mS14LwMsOeLO02JfYWLyc+ZKlolUcsCZiO9kpRd3Os8T+s6kSBw0Hz17SuVJBEWDuCk0= ceph-523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:30.181 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:07:30.181 INFO:teuthology.orchestra.run.smithi017.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T14:07:30.187 INFO:teuthology.orchestra.run.smithi017.stdout:Adding host smithi017... 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:29] ENGINE Bus STARTING 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:29] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:29] ENGINE Bus STARTED 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:29] ENGINE Error in HTTPServer.serve 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:07:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:07:30.952 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:07:30.952 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:07:30.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:30.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:30 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:31.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:31 smithi017 ceph-mon[29399]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi017", "addr": "172.21.15.17", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:33.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:32 smithi017 ceph-mon[29399]: Deploying cephadm binary to smithi017 2024-09-13T14:07:34.780 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Added host 'smithi017' with addr '172.21.15.17' 2024-09-13T14:07:34.780 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mon service... 2024-09-13T14:07:35.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:35 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:35.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:35 smithi017 ceph-mon[29399]: Added host smithi017 2024-09-13T14:07:35.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:35 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:35.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T14:07:35.994 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mgr service... 2024-09-13T14:07:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:36 smithi017 ceph-mon[29399]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:36 smithi017 ceph-mon[29399]: Saving service mon spec with placement count:5 2024-09-13T14:07:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:36 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:37.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T14:07:37.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:37 smithi017 ceph-mon[29399]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:37.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:37 smithi017 ceph-mon[29399]: Saving service mgr spec with placement count:2 2024-09-13T14:07:37.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:37 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:39 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3975603486' entity='client.admin' 2024-09-13T14:07:40.146 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling the dashboard module... 2024-09-13T14:07:40.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:40 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:40.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:40 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2247205676' entity='client.admin' 2024-09-13T14:07:40.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:40 smithi017 ceph-mon[29399]: from='mgr.14116 172.21.15.17:0/430238016' entity='mgr.y' 2024-09-13T14:07:41.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3402078236' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T14:07:41.557 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setuser ceph since I am not root 2024-09-13T14:07:41.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setgroup ceph since I am not root 2024-09-13T14:07:41.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:41.429+0000 7f509b8f1ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:07:41.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:41.486+0000 7f509b8f1ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:41.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:41.555+0000 7f509b8f1ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:07:41.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:41.686+0000 7f509b8f1ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:41.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:41.828+0000 7f509b8f1ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:07:42.104 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:42.103+0000 7f509b8f1ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:07:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:42 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3402078236' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T14:07:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:42 smithi017 ceph-mon[29399]: mgrmap e9: y(active, since 17s) 2024-09-13T14:07:42.374 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:42.170+0000 7f509b8f1ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:07:42.374 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:42.341+0000 7f509b8f1ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:43.122 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2024-09-13T14:07:43.123 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 9... 2024-09-13T14:07:43.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:43 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2458185891' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:07:43.375 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:43.234+0000 7f509b8f1ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:07:43.375 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:43.303+0000 7f509b8f1ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:07:44.360 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.055+0000 7f509b8f1ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:07:44.360 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.171+0000 7f509b8f1ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:07:44.360 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.297+0000 7f509b8f1ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:07:44.360 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.359+0000 7f509b8f1ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:07:44.621 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.490+0000 7f509b8f1ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:07:44.624 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.559+0000 7f509b8f1ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:07:44.624 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.620+0000 7f509b8f1ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:07:44.895 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.760+0000 7f509b8f1ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:07:44.895 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.826+0000 7f509b8f1ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:07:44.896 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.894+0000 7f509b8f1ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:07:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:44.978+0000 7f509b8f1ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:07:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:45.038+0000 7f509b8f1ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:07:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:45.097+0000 7f509b8f1ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:07:46.132 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:45.877+0000 7f509b8f1ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:07:46.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: Active manager daemon y restarted 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: Activating manager daemon y 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: mgrmap e10: y(active, starting, since 0.00311803s) 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: Manager daemon y is now available 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:07:46.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:47.714+0000 7f504c44f640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:07:47] ENGINE Error in HTTPServer.serve 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Traceback (most recent call last): 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._connections.run(self.expiration_interval) 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._run(expiration_interval) 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:07:47.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s = self.context.wrap_socket( 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: return self.sslsocket_class._create( 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self.do_handshake() 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._sslobj.do_handshake() 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:07:47.831 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:07:47.832 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:07:47.832 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T14:07:47.832 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:07:47.833 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:07:47.833 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 9 is available 2024-09-13T14:07:47.833 INFO:teuthology.orchestra.run.smithi017.stdout:Generating a dashboard self-signed certificate... 2024-09-13T14:07:48.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-mon[29399]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:07:48.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-mon[29399]: mgrmap e11: y(active, since 1.00566s) 2024-09-13T14:07:48.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-mon[29399]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:07:48.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:48.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:47 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:47] ENGINE Bus STARTING 2024-09-13T14:07:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:47] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:07:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:47] ENGINE Bus STARTED 2024-09-13T14:07:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: [13/Sep/2024:14:07:47] ENGINE Error in HTTPServer.serve 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:07:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:49.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:49 smithi017 ceph-mon[29399]: mgrmap e12: y(active, since 2s) 2024-09-13T14:07:49.541 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T14:07:49.542 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial admin user... 2024-09-13T14:07:50.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:50 smithi017 ceph-mon[29399]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:51.492 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$1tn7dRj1Yvkf4CkSbqAhVeE/N3zzBc3qcJZH9s9XeYLvJQKIoeb1K", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726236470, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T14:07:51.492 INFO:teuthology.orchestra.run.smithi017.stdout:Fetching dashboard port number... 2024-09-13T14:07:52.082 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:51 smithi017 ceph-mon[29399]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:07:52.082 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:51 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:52.665 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T14:07:52.676 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:07:52.676 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-13T14:07:52.684 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:07:52.684 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-13T14:07:52.684 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-13T14:07:52.684 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T14:07:52.686 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph Dashboard is now available at: 2024-09-13T14:07:52.686 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout: URL: https://smithi017.front.sepia.ceph.com:8443/ 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout: User: admin 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout: Password: 8issgv8qcm 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout:Saving cluster configuration to /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config directory 2024-09-13T14:07:52.687 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling autotune for osd_memory_target 2024-09-13T14:07:52.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:52 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/84954978' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T14:07:53.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:53 smithi017 ceph-mon[29399]: mgrmap e13: y(active, since 6s) 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: ceph telemetry on 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout:For more information see: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:07:55.086 INFO:teuthology.orchestra.run.smithi017.stdout:Bootstrap complete. 2024-09-13T14:07:55.121 INFO:tasks.cephadm:Fetching config... 2024-09-13T14:07:55.121 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:07:55.121 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T14:07:55.137 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T14:07:55.138 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:07:55.138 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T14:07:55.194 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T14:07:55.194 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:07:55.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T14:07:55.260 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T14:07:55.260 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:07:55.260 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T14:07:55.316 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T14:07:55.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKY7NoEyCCkyxmlFcbbbJsX+bBik7S//n6sd3aNF0+5lWuYd0TBob2Su58Iw0mKFbRpHKs8M6nhmFqTNqSa4+tlGuU4ybJMrMn/jWzFaR7ktxFeDglkNWXj8yxUMfif9gB3zgujIT+NT3yM9SifswNX55KeA8vxDOIrfzY7AD3S0zRwjjDkxXIE9NB5V/OohYF6YdNTFgsdS6zdIwgRIXLyQGXK9CdjRekBYM69yFQCYiCEbKoCvSp1HAemHK2tbMAkHNk43qTsBZYFAsbaG16v3kFkBGRb59BoHRQjIY4VW1tgCWkUE9tXHWHPp+2CtBsTQ1ODHNu2qaY1Ayw+A49Fxm03LnxphBQxIgrA71LT/HMIHSXe76SwXkSHCEge81KO/1nSIbEkQbpTKQJE8MoTw/XEq0kKo1K9cJxjxN9o0g2+ARxe5sMLl/AwH/mS14LwMsOeLO02JfYWLyc+ZKlolUcsCZiO9kpRd3Os8T+s6kSBw0Hz17SuVJBEWDuCk0= ceph-523f52e4-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:07:55.391 INFO:teuthology.orchestra.run.smithi017.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKY7NoEyCCkyxmlFcbbbJsX+bBik7S//n6sd3aNF0+5lWuYd0TBob2Su58Iw0mKFbRpHKs8M6nhmFqTNqSa4+tlGuU4ybJMrMn/jWzFaR7ktxFeDglkNWXj8yxUMfif9gB3zgujIT+NT3yM9SifswNX55KeA8vxDOIrfzY7AD3S0zRwjjDkxXIE9NB5V/OohYF6YdNTFgsdS6zdIwgRIXLyQGXK9CdjRekBYM69yFQCYiCEbKoCvSp1HAemHK2tbMAkHNk43qTsBZYFAsbaG16v3kFkBGRb59BoHRQjIY4VW1tgCWkUE9tXHWHPp+2CtBsTQ1ODHNu2qaY1Ayw+A49Fxm03LnxphBQxIgrA71LT/HMIHSXe76SwXkSHCEge81KO/1nSIbEkQbpTKQJE8MoTw/XEq0kKo1K9cJxjxN9o0g2+ARxe5sMLl/AwH/mS14LwMsOeLO02JfYWLyc+ZKlolUcsCZiO9kpRd3Os8T+s6kSBw0Hz17SuVJBEWDuCk0= ceph-523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:55.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKY7NoEyCCkyxmlFcbbbJsX+bBik7S//n6sd3aNF0+5lWuYd0TBob2Su58Iw0mKFbRpHKs8M6nhmFqTNqSa4+tlGuU4ybJMrMn/jWzFaR7ktxFeDglkNWXj8yxUMfif9gB3zgujIT+NT3yM9SifswNX55KeA8vxDOIrfzY7AD3S0zRwjjDkxXIE9NB5V/OohYF6YdNTFgsdS6zdIwgRIXLyQGXK9CdjRekBYM69yFQCYiCEbKoCvSp1HAemHK2tbMAkHNk43qTsBZYFAsbaG16v3kFkBGRb59BoHRQjIY4VW1tgCWkUE9tXHWHPp+2CtBsTQ1ODHNu2qaY1Ayw+A49Fxm03LnxphBQxIgrA71LT/HMIHSXe76SwXkSHCEge81KO/1nSIbEkQbpTKQJE8MoTw/XEq0kKo1K9cJxjxN9o0g2+ARxe5sMLl/AwH/mS14LwMsOeLO02JfYWLyc+ZKlolUcsCZiO9kpRd3Os8T+s6kSBw0Hz17SuVJBEWDuCk0= ceph-523f52e4-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:07:55.541 INFO:teuthology.orchestra.run.smithi189.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKY7NoEyCCkyxmlFcbbbJsX+bBik7S//n6sd3aNF0+5lWuYd0TBob2Su58Iw0mKFbRpHKs8M6nhmFqTNqSa4+tlGuU4ybJMrMn/jWzFaR7ktxFeDglkNWXj8yxUMfif9gB3zgujIT+NT3yM9SifswNX55KeA8vxDOIrfzY7AD3S0zRwjjDkxXIE9NB5V/OohYF6YdNTFgsdS6zdIwgRIXLyQGXK9CdjRekBYM69yFQCYiCEbKoCvSp1HAemHK2tbMAkHNk43qTsBZYFAsbaG16v3kFkBGRb59BoHRQjIY4VW1tgCWkUE9tXHWHPp+2CtBsTQ1ODHNu2qaY1Ayw+A49Fxm03LnxphBQxIgrA71LT/HMIHSXe76SwXkSHCEge81KO/1nSIbEkQbpTKQJE8MoTw/XEq0kKo1K9cJxjxN9o0g2+ARxe5sMLl/AwH/mS14LwMsOeLO02JfYWLyc+ZKlolUcsCZiO9kpRd3Os8T+s6kSBw0Hz17SuVJBEWDuCk0= ceph-523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:07:55.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T14:07:55.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:55 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3701663355' entity='client.admin' 2024-09-13T14:07:55.807 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:07:58.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:58.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi017", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/685095440' entity='client.admin' 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:07:58.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:07:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:07:58.636 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T14:07:58.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T14:07:58.886 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:01.117 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi189 2024-09-13T14:08:01.118 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:08:01.118 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.conf 2024-09-13T14:08:01.137 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:08:01.137 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:08:01.196 INFO:tasks.cephadm:Adding host smithi189 to orchestrator... 2024-09-13T14:08:01.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch host add smithi189 2024-09-13T14:08:01.454 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:01 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:03 smithi017 ceph-mon[29399]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:04 smithi017 ceph-mon[29399]: Deploying cephadm binary to smithi189 2024-09-13T14:08:06.460 INFO:teuthology.orchestra.run.smithi017.stdout:Added host 'smithi189' with addr '172.21.15.189' 2024-09-13T14:08:07.044 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:06 smithi017 ceph-mon[29399]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:07.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:06 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:07.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:06 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:07.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:06 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:07.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T14:08:07.298 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:08 smithi017 ceph-mon[29399]: Added host smithi189 2024-09-13T14:08:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:09.245 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:08:09.245 INFO:teuthology.orchestra.run.smithi017.stdout:[{"addr": "172.21.15.17", "hostname": "smithi017", "labels": [], "status": ""}, {"addr": "172.21.15.189", "hostname": "smithi189", "labels": [], "status": ""}] 2024-09-13T14:08:09.245 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:09 smithi017 ceph-mon[29399]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:09.802 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T14:08:09.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T14:08:10.054 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:10.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:10 smithi017 ceph-mon[29399]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:08:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:11.264 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:08:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:08:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:08:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:13.061 INFO:teuthology.orchestra.run.smithi017.stderr:adjusted tunables profile to default 2024-09-13T14:08:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:13 smithi017 ceph-mon[29399]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1501702722' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:08:13.775 INFO:tasks.cephadm:Adding mon.a on smithi017 2024-09-13T14:08:13.775 INFO:tasks.cephadm:Adding mon.c on smithi017 2024-09-13T14:08:13.775 INFO:tasks.cephadm:Adding mon.b on smithi189 2024-09-13T14:08:13.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b' 2024-09-13T14:08:14.019 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:14 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1501702722' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:08:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:14 smithi017 ceph-mon[29399]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:15 smithi017 ceph-mon[29399]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:15.796 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mon update... 2024-09-13T14:08:16.374 DEBUG:teuthology.orchestra.run.smithi017:mon.c> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T14:08:16.382 DEBUG:teuthology.orchestra.run.smithi189:mon.b> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T14:08:16.384 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:08:16.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:08:17.157 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b;count:3 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: Deploying daemon mon.b on smithi189 2024-09-13T14:08:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:16 smithi017 ceph-mon[29399]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:18 smithi017 ceph-mon[29399]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:20.266 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:08:20.266 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:06:53.813281Z","created":"2024-09-13T14:06:53.813281Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T14:08:20.266 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-09-13T14:08:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:20 smithi189 systemd[1]: Starting Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:08:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:20 smithi017 ceph-mon[29399]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:20 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/2015788083' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:21.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 podman[33257]: 2024-09-13 14:08:21.091012714 +0000 UTC m=+0.112461490 container create 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T14:08:21.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 podman[33257]: 2024-09-13 14:08:20.994294091 +0000 UTC m=+0.015742883 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 podman[33257]: 2024-09-13 14:08:21.29363954 +0000 UTC m=+0.315088322 container init 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS) 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 podman[33257]: 2024-09-13 14:08:21.298015287 +0000 UTC m=+0.319464062 container start 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: pidfile_write: ignore empty --pid-file 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: load: jerasure load: lrc 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: DB SUMMARY 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: DB Session ID: UI9X3O6Q3U89QH9HIJE1 2024-09-13T14:08:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.env: 0x55d990827440 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.fs: Posix File System 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.info_log: 0x55d993062ec0 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.statistics: (nil) 2024-09-13T14:08:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.use_fsync: 0 2024-09-13T14:08:21.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:08:21.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.db_log_dir: 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:08:21.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.write_buffer_manager: 0x55d9931207e0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:08:21.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.unordered_write: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.row_cache: None 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.wal_filter: None 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:08:21.762 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:08:21.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_open_files: -1 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:08:21.764 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Compression algorithms supported: 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kZSTD supported: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:08:21.765 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.merge_operator: 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_filter: None 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d993009a38) 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:08:21.766 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_type: 0 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: data_block_index_type: 0 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_shortening: 1 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: hash_index_allow_collision: 1 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: checksum: 1 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: no_block_cache: 0 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache: 0x55d99303f1f0 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_options: 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: capacity : 536870912 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: num_shard_bits : 4 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: strict_capacity_limit : 0 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_compressed: (nil) 2024-09-13T14:08:21.767 INFO:journalctl@ceph.mon.b.smithi189.stdout: persistent_cache: (nil) 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_size: 4096 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_size_deviation: 10 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_restart_interval: 16 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_block_restart_interval: 1 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: metadata_block_size: 4096 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: partition_filters: 0 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: use_delta_encoding: 1 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: whole_key_filtering: 1 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: verify_compression: 0 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: format_version: 4 2024-09-13T14:08:21.768 INFO:journalctl@ceph.mon.b.smithi189.stdout: enable_index_compression: 1 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_align: 0 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression: NoCompression 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.num_levels: 7 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:08:21.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:08:21.770 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:08:21.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:08:21.772 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:08:21.773 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.ttl: 2592000 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236501322200, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:08:21.774 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236501322502, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236501, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fe899aa2-5a2d-4184-b423-6aa05d9b8818", "db_session_id": "UI9X3O6Q3U89QH9HIJE1"}} 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236501322968, "job": 1, "event": "recovery_finished"} 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d993072700 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: DB pointer 0x55d993042000 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** DB Stats ** 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:08:21.775 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:21.776 INFO:journalctl@ceph.mon.b.smithi189.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:08:21.777 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:21.778 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: using public_addr v2:172.21.15.189:0/0 -> [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: starting mon.b rank -1 at public addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] at bind addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(???) e0 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).mds e1 new map 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T14:08:21.779 INFO:journalctl@ceph.mon.b.smithi189.stdout: e1 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout: legacy client fscid: -1 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout: No filesystems configured 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:21.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1501702722' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b;count:3 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:21.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: Deploying daemon mon.b on smithi189 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/2015788083' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 ceph-mon[33294]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 bash[33257]: 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb 2024-09-13T14:08:21.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:21 smithi189 systemd[1]: Started Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:08:21.866 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:08:21.866 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:08:22.109 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: mon.a calling monitor election 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: mon.b calling monitor election 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: fsmap 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: mgrmap e13: y(active, since 40s) 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: overall HEALTH_OK 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:26.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:26 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: mon.a calling monitor election 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: mon.b calling monitor election 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: fsmap 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: mgrmap e13: y(active, since 40s) 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: overall HEALTH_OK 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:26 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:26.892 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:08:26.892 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":2,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:08:21.351292Z","created":"2024-09-13T14:06:53.813281Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:3300","nonce":0},{"type":"v1","addr":"172.21.15.189:6789","nonce":0}]},"addr":"172.21.15.189:6789/0","public_addr":"172.21.15.189:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T14:08:26.892 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 2 2024-09-13T14:08:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:27 smithi189 ceph-mon[33294]: Deploying daemon mon.c on smithi017 2024-09-13T14:08:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:27 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/893462930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:27 smithi017 ceph-mon[29399]: Deploying daemon mon.c on smithi017 2024-09-13T14:08:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:27 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/893462930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:28.423 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:08:28.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:08:28.665 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:08:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:28 smithi017 ceph-mon[29399]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:28 smithi189 ceph-mon[33294]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 podman[38218]: 2024-09-13 14:08:29.238454889 +0000 UTC m=+0.016096157 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:08:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 podman[38218]: 2024-09-13 14:08:29.559638195 +0000 UTC m=+0.337279453 container init 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:08:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 podman[38218]: 2024-09-13 14:08:29.56449797 +0000 UTC m=+0.342139223 container start 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:08:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:08:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pidfile_write: ignore empty --pid-file 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: load: jerasure load: lrc 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: RocksDB version: 6.15.5 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Compile date Sep 12 2024 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: DB SUMMARY 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: DB Session ID: 7FOV9EEU5VEXQML4ZZ5F 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-13T14:08:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.env: 0x55ba4fba8440 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.fs: Posix File System 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.info_log: 0x55ba5124cec0 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.statistics: (nil) 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.use_fsync: 0 2024-09-13T14:08:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.db_log_dir: 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T14:08:29.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.write_buffer_manager: 0x55ba5130a7b0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:08:29.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.unordered_write: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.row_cache: None 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.wal_filter: None 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.preserve_deletes: 0 2024-09-13T14:08:29.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:08:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_open_files: -1 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:08:29.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Compression algorithms supported: 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kZSTD supported: 0 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T14:08:29.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.merge_operator: 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_filter: None 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ba511f3a38) 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2024-09-13T14:08:29.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: hash_index_allow_collision: 1 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 1 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x55ba512291f0 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2024-09-13T14:08:29.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2024-09-13T14:08:29.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 4 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression: NoCompression 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.num_levels: 7 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:08:29.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:08:29.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:08:29.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:08:29.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:08:29.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.ttl: 2592000 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:08:29.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236509589044, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236509589358, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236509, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "37d4083c-f79a-49bf-8b63-e1fabc386720", "db_session_id": "7FOV9EEU5VEXQML4ZZ5F"}} 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236509589700, "job": 1, "event": "recovery_finished"} 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55ba5125c700 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: DB pointer 0x55ba5122c000 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:29.969 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:29.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:08:29.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:08:29.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: using public_addrv [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: starting mon.c rank -1 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(???) e0 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: e1 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: -1 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: No filesystems configured 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:08:29.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1501702722' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi189:172.21.15.189=b;count:3 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:29.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: Deploying daemon mon.b on smithi189 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/2015788083' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.a calling monitor election 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.b calling monitor election 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: fsmap 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mgrmap e13: y(active, since 40s) 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: overall HEALTH_OK 2024-09-13T14:08:29.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: Deploying daemon mon.c on smithi017 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/893462930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 ceph-mon[38255]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 bash[38218]: 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b 2024-09-13T14:08:29.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:29 smithi017 systemd[1]: Started Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:08:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: mon.a calling monitor election 2024-09-13T14:08:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: mon.b calling monitor election 2024-09-13T14:08:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: mon.c calling monitor election 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: fsmap 2024-09-13T14:08:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: mgrmap e13: y(active, since 48s) 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: overall HEALTH_OK 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: mon.a calling monitor election 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: mon.b calling monitor election 2024-09-13T14:08:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: mon.c calling monitor election 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: fsmap 2024-09-13T14:08:34.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: mgrmap e13: y(active, since 48s) 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: overall HEALTH_OK 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:34 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:34.999 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:08:34.999 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":3,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:08:29.611288Z","created":"2024-09-13T14:06:53.813281Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:3300","nonce":0},{"type":"v1","addr":"172.21.15.189:6789","nonce":0}]},"addr":"172.21.15.189:6789/0","public_addr":"172.21.15.189:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3301","nonce":0},{"type":"v1","addr":"172.21.15.17:6790","nonce":0}]},"addr":"172.21.15.17:6790/0","public_addr":"172.21.15.17:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-13T14:08:34.999 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 3 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: mon.a calling monitor election 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: mon.b calling monitor election 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: mon.c calling monitor election 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.000 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: fsmap 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: mgrmap e13: y(active, since 48s) 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: overall HEALTH_OK 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:35.001 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:34 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:35.641 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T14:08:35.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T14:08:35.900 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/1005596775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:35.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/1005596775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:35.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:35 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/1005596775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:08:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:37.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:36 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:36 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:37.720 INFO:teuthology.orchestra.run.smithi017.stdout:# minimal ceph.conf for 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:37.720 INFO:teuthology.orchestra.run.smithi017.stdout:[global] 2024-09-13T14:08:37.720 INFO:teuthology.orchestra.run.smithi017.stdout: fsid = 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:37.720 INFO:teuthology.orchestra.run.smithi017.stdout: mon_host = [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2474250613' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:08:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2474250613' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:08:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2474250613' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:38.721 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T14:08:38.722 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:08:38.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:08:38.752 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:08:38.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:08:38.820 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:08:38.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:08:38.853 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:08:38.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:08:38.925 INFO:tasks.cephadm:Adding mgr.y on smithi017 2024-09-13T14:08:38.926 INFO:tasks.cephadm:Adding mgr.x on smithi189 2024-09-13T14:08:38.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi017=y;smithi189=x' 2024-09-13T14:08:38.993 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:38 smithi017 ceph-mon[29399]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:38.994 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:38 smithi017 ceph-mon[38255]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:39.205 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:08:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:38 smithi189 ceph-mon[33294]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:40.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:08:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:40 smithi189 ceph-mon[33294]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:08:40.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:08:40.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:40.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[38255]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:40 smithi017 ceph-mon[29399]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:08:40.870 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mgr update... 2024-09-13T14:08:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:41.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:41.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:41.987 DEBUG:teuthology.orchestra.run.smithi189:mgr.x> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T14:08:41.989 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T14:08:41.989 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T14:08:41.990 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:08:41.990 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:08:42.010 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:08:42.010 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_1 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:42.070 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-13 14:08:29.060279153 +0000 2024-09-13T14:08:42.071 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-13 14:03:44.897216666 +0000 2024-09-13T14:08:42.071 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-13 14:03:44.897216666 +0000 2024-09-13T14:08:42.071 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-13 14:03:44.897216666 +0000 2024-09-13T14:08:42.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:08:42.141 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-13T14:08:42.141 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-13T14:08:42.142 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000132704 s, 3.9 MB/s 2024-09-13T14:08:42.143 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:08:42.202 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_2 2024-09-13T14:08:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:42 smithi189 ceph-mon[33294]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:08:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:42 smithi189 ceph-mon[33294]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:08:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:42 smithi189 ceph-mon[33294]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:42 smithi189 ceph-mon[33294]: Saving service mgr spec with placement smithi017=y;smithi189=x;count:2 2024-09-13T14:08:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:08:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-13 14:08:29.060279153 +0000 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-13 14:03:44.896216677 +0000 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-13 14:03:44.896216677 +0000 2024-09-13T14:08:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-13 14:03:44.896216677 +0000 2024-09-13T14:08:42.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:08:42.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[38255]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:08:42.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[38255]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:08:42.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[38255]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:42.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[38255]: Saving service mgr spec with placement smithi017=y;smithi189=x;count:2 2024-09-13T14:08:42.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[29399]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:08:42.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[29399]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:08:42.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[29399]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:42.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:42 smithi017 ceph-mon[29399]: Saving service mgr spec with placement smithi017=y;smithi189=x;count:2 2024-09-13T14:08:42.330 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-13T14:08:42.330 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-13T14:08:42.330 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000115762 s, 4.4 MB/s 2024-09-13T14:08:42.331 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:08:42.391 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_3 2024-09-13T14:08:42.450 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:08:42.450 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:42.450 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 962 Links: 1 2024-09-13T14:08:42.450 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:42.451 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:42.451 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-13 14:08:29.060279153 +0000 2024-09-13T14:08:42.451 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.451 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.451 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:08:42.520 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-13T14:08:42.520 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-13T14:08:42.520 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000138292 s, 3.7 MB/s 2024-09-13T14:08:42.521 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:08:42.580 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_4 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 963 Links: 1 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-13 14:08:29.061279142 +0000 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.639 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-13 14:03:44.895216687 +0000 2024-09-13T14:08:42.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:08:42.706 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-13T14:08:42.706 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-13T14:08:42.706 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000106462 s, 4.8 MB/s 2024-09-13T14:08:42.707 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:08:42.767 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:08:42.767 DEBUG:teuthology.orchestra.run.smithi189:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:08:42.783 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:08:42.783 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_1 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:42.839 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-09-13 14:08:20.318425350 +0000 2024-09-13T14:08:42.840 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:42.840 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:42.840 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:42.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:08:42.904 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-09-13T14:08:42.904 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-09-13T14:08:42.904 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.00018934 s, 2.7 MB/s 2024-09-13T14:08:42.905 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:08:42.963 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_2 2024-09-13T14:08:43.018 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:08:43.018 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-09-13 14:08:20.318425350 +0000 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-09-13 14:03:45.586780936 +0000 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-09-13 14:03:45.586780936 +0000 2024-09-13T14:08:43.019 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-09-13 14:03:45.586780936 +0000 2024-09-13T14:08:43.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:08:43.083 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-09-13T14:08:43.083 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-09-13T14:08:43.083 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000211456 s, 2.4 MB/s 2024-09-13T14:08:43.085 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:08:43.142 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_3 2024-09-13T14:08:43.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:43.186 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.198 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:08:43.198 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 968 Links: 1 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-09-13 14:08:20.319425331 +0000 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-09-13 14:03:45.584780975 +0000 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-09-13 14:03:45.584780975 +0000 2024-09-13T14:08:43.199 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-09-13 14:03:45.584780975 +0000 2024-09-13T14:08:43.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:08:43.263 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-09-13T14:08:43.263 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-09-13T14:08:43.263 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000231848 s, 2.2 MB/s 2024-09-13T14:08:43.264 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:08:43.321 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_4 2024-09-13T14:08:43.377 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:08:43.377 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:08:43.377 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-09-13 14:08:20.319425331 +0000 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:43.378 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-09-13 14:03:45.585780956 +0000 2024-09-13T14:08:43.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:08:43.441 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-09-13T14:08:43.441 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-09-13T14:08:43.441 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000190755 s, 2.7 MB/s 2024-09-13T14:08:43.442 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:43.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:43.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:43.499 INFO:tasks.cephadm:Deploying osd.0 on smithi017 with /dev/vg_nvme/lv_4... 2024-09-13T14:08:43.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:08:43.758 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:44 smithi017 ceph-mon[29399]: Deploying daemon mgr.x on smithi189 2024-09-13T14:08:44.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:44 smithi017 ceph-mon[38255]: Deploying daemon mgr.x on smithi189 2024-09-13T14:08:44.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:44 smithi189 ceph-mon[33294]: Deploying daemon mgr.x on smithi189 2024-09-13T14:08:44.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 podman[35320]: 2024-09-13 14:08:44.448698631 +0000 UTC m=+0.130976244 container create 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:08:45.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 podman[35320]: 2024-09-13 14:08:44.676184333 +0000 UTC m=+0.358461883 container init 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T14:08:45.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 podman[35320]: 2024-09-13 14:08:44.681257828 +0000 UTC m=+0.363535379 container start 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 ceph-mgr[35358]: -- 172.21.15.189:0/3295739625 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dfa4388340 con 0x55dfa4394c00 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 bash[35320]: 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 systemd[1]: Started Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:44.793+0000 7f300f598ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:44.852+0000 7f300f598ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:08:45.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:44.919+0000 7f300f598ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:45 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:45.476 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:45.050+0000 7f300f598ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:08:45.476 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:45.198+0000 7f300f598ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:08:45.476 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:45.474+0000 7f300f598ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:45.477 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:45.738 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:45.538+0000 7f300f598ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:08:45.738 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:45.707+0000 7f300f598ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:08:45.985 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:08:46.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi017:vg_nvme/lv_4 2024-09-13T14:08:46.268 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:08:46.919 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:46.564+0000 7f300f598ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:08:46.920 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:46.627+0000 7f300f598ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:08:47.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:47.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[38255]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:46 smithi017 ceph-mon[29399]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:08:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:46 smithi189 ceph-mon[33294]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:08:47.624 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.334+0000 7f300f598ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:08:47.624 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.445+0000 7f300f598ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:08:47.624 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.564+0000 7f300f598ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:08:48.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.622+0000 7f300f598ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:08:48.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.746+0000 7f300f598ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:08:48.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.813+0000 7f300f598ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:08:48.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:47.869+0000 7f300f598ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:08:48.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.003+0000 7f300f598ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:08:48.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.065+0000 7f300f598ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:08:48.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.129+0000 7f300f598ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:08:48.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.208+0000 7f300f598ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:08:48.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.266+0000 7f300f598ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:08:48.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:48.323+0000 7f300f598ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:08:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[29399]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[38255]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:49.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:08:49.067+0000 7f300f598ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:08:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-mon[33294]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:08:49.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:49.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:08:49.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:08:49.986 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: Standby manager daemon x started 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:08:49.987 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:49 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:08:50.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:08:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:49 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/422284828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:08:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[29399]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[29399]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-13T14:08:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:08:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[38255]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[38255]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-13T14:08:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:51 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:08:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:51 smithi189 ceph-mon[33294]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:51 smithi189 ceph-mon[33294]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-13T14:08:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:51 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:08:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]: dispatch 2024-09-13T14:08:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]': finished 2024-09-13T14:08:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[29399]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:08:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:08:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2468792864' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:08:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]: dispatch 2024-09-13T14:08:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]': finished 2024-09-13T14:08:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[38255]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:08:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:08:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:52 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2468792864' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:08:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:52 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]: dispatch 2024-09-13T14:08:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:52 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3920128679' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b9a98879-a576-4dbf-80ce-247ecc208028"}]': finished 2024-09-13T14:08:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:52 smithi189 ceph-mon[33294]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:08:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:08:52.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:52 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2468792864' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:08:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:53 smithi017 ceph-mon[38255]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:53 smithi017 ceph-mon[29399]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:53 smithi189 ceph-mon[33294]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:55.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:55 smithi017 ceph-mon[38255]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:55.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:55 smithi017 ceph-mon[29399]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:55 smithi189 ceph-mon[33294]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:57.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:57 smithi017 ceph-mon[29399]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:57.346 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:57 smithi017 ceph-mon[38255]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:57.571 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:57 smithi189 ceph-mon[33294]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:08:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:08:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:08:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:08:59.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:59 smithi017 ceph-mon[29399]: Deploying daemon osd.0 on smithi017 2024-09-13T14:08:59.516 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:08:59 smithi017 ceph-mon[29399]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:59.516 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:59 smithi017 ceph-mon[38255]: Deploying daemon osd.0 on smithi017 2024-09-13T14:08:59.516 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:08:59 smithi017 ceph-mon[38255]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:08:59.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:59 smithi189 ceph-mon[33294]: Deploying daemon osd.0 on smithi017 2024-09-13T14:08:59.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:08:59 smithi189 ceph-mon[33294]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:01.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:01 smithi017 ceph-mon[29399]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:01.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:01 smithi017 ceph-mon[38255]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:01 smithi189 ceph-mon[33294]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:02.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:02.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:02.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:03.666 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:03 smithi017 ceph-mon[29399]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:03.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:03 smithi017 ceph-mon[38255]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:03 smithi189 ceph-mon[33294]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:05.234 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 0 on host 'smithi017' 2024-09-13T14:09:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:09:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:05.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:05.842 DEBUG:teuthology.orchestra.run.smithi017:osd.0> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T14:09:05.844 INFO:tasks.cephadm:Deploying osd.1 on smithi017 with /dev/vg_nvme/lv_3... 2024-09-13T14:09:05.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:09:06.666 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:06.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[29399]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:09:06.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[29399]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:09:06.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:06.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[29399]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:06.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[29399]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:06.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[38255]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:09:06.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[38255]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:09:06.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:06.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[38255]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:06.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-mon[38255]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:06.775 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[42619]: 2024-09-13T14:09:06.559+0000 7f61b53b1640 -1 osd.0 0 waiting for initial osdmap 2024-09-13T14:09:06.775 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[42619]: 2024-09-13T14:09:06.561+0000 7f61ae1e4640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:09:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:06 smithi189 ceph-mon[33294]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:09:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:06 smithi189 ceph-mon[33294]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:09:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:06 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:06 smithi189 ceph-mon[33294]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:06 smithi189 ceph-mon[33294]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:09:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[29399]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[29399]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:09:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[38255]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[38255]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:09:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:07 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:07 smithi189 ceph-mon[33294]: from='osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:07 smithi189 ceph-mon[33294]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:09:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:07 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:07 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241] boot 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:08 smithi189 ceph-mon[33294]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:09:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:09:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241] boot 2024-09-13T14:09:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:09:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[29399]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:08.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:09:08.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:09:08.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: osd.0 [v2:172.21.15.17:6802/2866456241,v1:172.21.15.17:6803/2866456241] boot 2024-09-13T14:09:08.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:09:08.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:09:08.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:08 smithi017 ceph-mon[38255]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi017 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:09.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi017 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-13T14:09:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi017 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:10.577 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:09:10.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:10 smithi017 ceph-mon[29399]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:10.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:10 smithi017 ceph-mon[38255]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:10.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi017:vg_nvme/lv_3 2024-09-13T14:09:10.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:10 smithi189 ceph-mon[33294]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:13.239 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[38255]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:13.239 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:13.239 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:13.240 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:13.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[29399]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:13.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:13.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:13.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:12 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:12 smithi189 ceph-mon[33294]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:12 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:12 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:12 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:13.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:13 smithi017 ceph-mon[29399]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:13.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:13 smithi017 ceph-mon[38255]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:14.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:13 smithi189 ceph-mon[33294]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:15.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/22643568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]': finished 2024-09-13T14:09:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:09:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:14 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/22643568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]': finished 2024-09-13T14:09:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/22643568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]: dispatch 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2"}]': finished 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:09:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:14 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:16.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:15 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1818514761' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:15 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1818514761' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:15 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1818514761' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:16 smithi189 ceph-mon[33294]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:16 smithi017 ceph-mon[29399]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:16 smithi017 ceph-mon[38255]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:19.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:18 smithi189 ceph-mon[33294]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:19.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:18 smithi017 ceph-mon[29399]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:19.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:18 smithi017 ceph-mon[38255]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[29399]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:09:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[38255]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:09:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:20 smithi189 ceph-mon[33294]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:09:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:21 smithi017 ceph-mon[29399]: Deploying daemon osd.1 on smithi017 2024-09-13T14:09:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:21 smithi017 ceph-mon[38255]: Deploying daemon osd.1 on smithi017 2024-09-13T14:09:22.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:21 smithi189 ceph-mon[33294]: Deploying daemon osd.1 on smithi017 2024-09-13T14:09:23.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:22 smithi189 ceph-mon[33294]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:23.288 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:22 smithi017 ceph-mon[29399]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:23.289 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:22 smithi017 ceph-mon[38255]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:24 smithi189 ceph-mon[33294]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:25.284 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:24 smithi017 ceph-mon[29399]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:25.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:24 smithi017 ceph-mon[38255]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:25 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:26.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:25 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:26.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:25 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:26.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:25 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:27.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:26 smithi017 ceph-mon[38255]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:27.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:26 smithi017 ceph-mon[29399]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:26 smithi189 ceph-mon[33294]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:28.619 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 1 on host 'smithi017' 2024-09-13T14:09:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:28 smithi189 ceph-mon[33294]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:28 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:28 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:28 smithi189 ceph-mon[33294]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[29399]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[29399]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[38255]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:29.263 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:28 smithi017 ceph-mon[38255]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:09:29.292 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:09:29.294 INFO:tasks.cephadm:Deploying osd.2 on smithi017 with /dev/vg_nvme/lv_2... 2024-09-13T14:09:29.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:09:29.602 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[29399]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:30.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-mon[38255]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:30.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[48981]: 2024-09-13T14:09:30.627+0000 7f44c8a24640 -1 osd.1 0 waiting for initial osdmap 2024-09-13T14:09:30.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[48981]: 2024-09-13T14:09:30.630+0000 7f44c2859640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:09:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:09:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:09:30.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:30.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:30.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:30 smithi189 ceph-mon[33294]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:09:31.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[29399]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:31.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[29399]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[38255]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[38255]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:31.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:31 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:31 smithi189 ceph-mon[33294]: from='osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:31 smithi189 ceph-mon[33294]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:09:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:31 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:31 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123] boot 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[38255]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123] boot 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:32 smithi017 ceph-mon[29399]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:32.706 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:09:32.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi017:vg_nvme/lv_2 2024-09-13T14:09:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:09:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:09:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: osd.1 [v2:172.21.15.17:6810/3915572123,v1:172.21.15.17:6811/3915572123] boot 2024-09-13T14:09:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:09:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:09:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:32 smithi189 ceph-mon[33294]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:33.245 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:33 smithi017 ceph-mon[29399]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:09:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:33 smithi017 ceph-mon[38255]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:09:34.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:33 smithi189 ceph-mon[33294]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:09:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:34 smithi017 ceph-mon[29399]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:34 smithi017 ceph-mon[38255]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:34 smithi189 ceph-mon[33294]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi017 2024-09-13T14:09:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi017 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:36.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi017 2024-09-13T14:09:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:09:36.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:09:36.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:37.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:36 smithi017 ceph-mon[29399]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:37.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:36 smithi017 ceph-mon[29399]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:36 smithi017 ceph-mon[38255]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:36 smithi017 ceph-mon[38255]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:36 smithi189 ceph-mon[33294]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:09:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:36 smithi189 ceph-mon[33294]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]: dispatch 2024-09-13T14:09:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]': finished 2024-09-13T14:09:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[29399]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:09:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]: dispatch 2024-09-13T14:09:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]': finished 2024-09-13T14:09:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[38255]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:09:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]: dispatch 2024-09-13T14:09:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3559129310' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb11f41d-35bc-4180-a0f6-781c7dbe5b5b"}]': finished 2024-09-13T14:09:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:37 smithi189 ceph-mon[33294]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:09:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:38 smithi017 ceph-mon[29399]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:38 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4224909456' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:38 smithi017 ceph-mon[38255]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:38 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4224909456' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:38 smithi189 ceph-mon[33294]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:38 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4224909456' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:09:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:40 smithi017 ceph-mon[29399]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:40 smithi017 ceph-mon[38255]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:40 smithi189 ceph-mon[33294]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:42 smithi017 ceph-mon[29399]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:42 smithi017 ceph-mon[38255]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:42 smithi189 ceph-mon[33294]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:44.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:09:44.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:44.232 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:09:44.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:09:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:45.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:44 smithi017 ceph-mon[29399]: Deploying daemon osd.2 on smithi017 2024-09-13T14:09:45.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:44 smithi017 ceph-mon[29399]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:45.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:44 smithi017 ceph-mon[38255]: Deploying daemon osd.2 on smithi017 2024-09-13T14:09:45.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:44 smithi017 ceph-mon[38255]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:45.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:44 smithi189 ceph-mon[33294]: Deploying daemon osd.2 on smithi017 2024-09-13T14:09:45.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:44 smithi189 ceph-mon[33294]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:47.216 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:46 smithi017 ceph-mon[29399]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:47.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:46 smithi017 ceph-mon[38255]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:46 smithi189 ceph-mon[33294]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:48 smithi189 ceph-mon[33294]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[29399]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[38255]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:49.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:49.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:50 smithi189 ceph-mon[33294]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:51.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:50 smithi017 ceph-mon[29399]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:51.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:50 smithi017 ceph-mon[38255]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:51.701 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 2 on host 'smithi017' 2024-09-13T14:09:52.363 DEBUG:teuthology.orchestra.run.smithi017:osd.2> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T14:09:52.365 INFO:tasks.cephadm:Deploying osd.3 on smithi017 with /dev/vg_nvme/lv_1... 2024-09-13T14:09:52.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:09:52.689 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:09:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:53.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[38255]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:09:53.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[38255]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:09:53.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:53.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[38255]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[38255]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[29399]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[29399]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[29399]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:53.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-mon[29399]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:53.833 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[55364]: 2024-09-13T14:09:53.709+0000 7f9929ecc640 -1 osd.2 0 waiting for initial osdmap 2024-09-13T14:09:53.834 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:09:53 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[55364]: 2024-09-13T14:09:53.712+0000 7f9925d05640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:09:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:53 smithi189 ceph-mon[33294]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:09:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:53 smithi189 ceph-mon[33294]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:09:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:53 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:53 smithi189 ceph-mon[33294]: from='osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:53 smithi189 ceph-mon[33294]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:09:54.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[29399]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[29399]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[29399]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[38255]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[38255]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:54.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:54 smithi017 ceph-mon[38255]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:54 smithi189 ceph-mon[33294]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:09:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:54 smithi189 ceph-mon[33294]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:09:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:54 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:54 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:54 smithi189 ceph-mon[33294]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:09:55.399 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:09:55.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi017:vg_nvme/lv_1 2024-09-13T14:09:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:55 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:09:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:55 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:09:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:55 smithi189 ceph-mon[33294]: osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177] boot 2024-09-13T14:09:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:55 smithi189 ceph-mon[33294]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:09:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:56.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:09:56.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:09:56.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[38255]: osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177] boot 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[38255]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[29399]: osd.2 [v2:172.21.15.17:6818/1775042177,v1:172.21.15.17:6819/1775042177] boot 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[29399]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:09:56.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:09:56.445 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[29399]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[29399]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[38255]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[38255]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:56.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:56 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:09:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:56 smithi189 ceph-mon[33294]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:09:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:56 smithi189 ceph-mon[33294]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:56 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:09:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:57 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:09:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:57 smithi189 ceph-mon[33294]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:09:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:57 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:09:58.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:09:58.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[29399]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:09:58.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:09:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:09:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[38255]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:09:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:57 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:09:58.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:58 smithi017 sudo[59432]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:09:58.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:58 smithi017 sudo[59432]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:09:58.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:58 smithi017 sudo[59432]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:09:58.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:58 smithi017 sudo[59432]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi017 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[29399]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi017 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:58.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:58.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:58.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:58.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:58.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:58 smithi017 ceph-mon[38255]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:09:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi017 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:09:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:09:59.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:09:59.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:09:59.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:58 smithi189 ceph-mon[33294]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:09:59.444 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59530]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:09:59.444 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59530]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:09:59.444 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59530]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:09:59.444 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59530]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:09:59.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59599]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:09:59.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59599]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:09:59.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59599]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:09:59.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:09:59 smithi017 sudo[59599]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:00.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:10:00.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:00.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:00.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:00.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:09:59 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:00.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:10:00.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:00.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:00.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:00.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:00.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:09:59 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:00.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59610]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:10:00.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59610]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:00.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59610]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:00.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59610]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:00.788 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59634]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:10:00.789 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59634]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:00.789 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59634]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:00.789 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:00 smithi017 sudo[59634]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:01.073 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:00 smithi017 ceph-mon[29399]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:01.073 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:00 smithi017 ceph-mon[29399]: overall HEALTH_OK 2024-09-13T14:10:01.074 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59674]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:10:01.074 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59674]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:01.074 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59674]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:01.074 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59674]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:01.074 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:00 smithi017 ceph-mon[38255]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:01.075 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:00 smithi017 ceph-mon[38255]: overall HEALTH_OK 2024-09-13T14:10:01.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:00 smithi189 ceph-mon[33294]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:01.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:00 smithi189 ceph-mon[33294]: overall HEALTH_OK 2024-09-13T14:10:01.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59707]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:10:01.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59707]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:01.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59707]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:01.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59707]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:02.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:01 smithi017 ceph-mon[38255]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:10:02.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59710]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:10:02.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59710]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:02.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59710]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:02.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 sudo[59710]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:02.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:01 smithi017 ceph-mon[29399]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:10:02.173 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:01 smithi189 ceph-mon[33294]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:10:02.174 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36010]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:10:02.174 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36010]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:02.174 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36010]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:02.174 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36010]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36013]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:10:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36013]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36013]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:02 smithi189 sudo[36013]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:02 smithi017 sudo[59747]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:10:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:02 smithi017 sudo[59747]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:02 smithi017 sudo[59747]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:02 smithi017 sudo[59747]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 sudo[59750]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:10:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 sudo[59750]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:10:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 sudo[59750]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:10:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 sudo[59750]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2788622626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]': finished 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2788622626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]': finished 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2788622626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:03.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]: dispatch 2024-09-13T14:10:03.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76cf0d9c-0405-4601-9c2c-3cbbba55dab8"}]': finished 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:03.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:03.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:03.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:03.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:10:03.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:10:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[29399]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4288507602' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[38255]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:04 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4288507602' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:04 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:10:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:04 smithi189 ceph-mon[33294]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:10:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:04 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4288507602' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:05.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:05 smithi017 ceph-mon[29399]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:05.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:05 smithi017 ceph-mon[38255]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:05 smithi189 ceph-mon[33294]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:07 smithi017 ceph-mon[29399]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:07 smithi017 ceph-mon[38255]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:07 smithi189 ceph-mon[33294]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:09.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[38255]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:10:09.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:10:09.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:09.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[29399]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:10:09.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:10:09.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:09 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:09 smithi189 ceph-mon[33294]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:10:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:10:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:09 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:10.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:10 smithi017 ceph-mon[29399]: Deploying daemon osd.3 on smithi017 2024-09-13T14:10:10.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:10 smithi017 ceph-mon[38255]: Deploying daemon osd.3 on smithi017 2024-09-13T14:10:10.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:10 smithi189 ceph-mon[33294]: Deploying daemon osd.3 on smithi017 2024-09-13T14:10:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:11 smithi017 ceph-mon[38255]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:10:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:11 smithi017 ceph-mon[29399]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:10:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:11 smithi189 ceph-mon[33294]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:10:13.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:13 smithi017 ceph-mon[29399]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:10:13.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:13 smithi017 ceph-mon[38255]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:10:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:13 smithi189 ceph-mon[33294]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:10:14.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:14.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:14.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:14.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:14.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:14.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:14 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:14 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:14 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:14 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:15 smithi017 ceph-mon[29399]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:15 smithi017 ceph-mon[38255]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:15 smithi189 ceph-mon[33294]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:16.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:16 smithi017 ceph-mon[29399]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:16.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:16 smithi017 ceph-mon[38255]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:16 smithi189 ceph-mon[33294]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:10:17.244 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 3 on host 'smithi017' 2024-09-13T14:10:17.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:17.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:10:17.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:17.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:17 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.851 DEBUG:teuthology.orchestra.run.smithi017:osd.3> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T14:10:17.853 INFO:tasks.cephadm:Deploying osd.4 on smithi189 with /dev/vg_nvme/lv_4... 2024-09-13T14:10:17.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:10:17.881 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:17.882 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:17 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:18.104 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:10:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:18 smithi189 ceph-mon[33294]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:10:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:18 smithi189 ceph-mon[33294]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:10:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:18 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:18.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:18 smithi189 ceph-mon[33294]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:10:18.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:18 smithi189 ceph-mon[33294]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[29399]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:10:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[29399]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:10:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[29399]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[29399]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[38255]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[38255]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[38255]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-13T14:10:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-mon[38255]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:10:18.951 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[61509]: 2024-09-13T14:10:18.628+0000 7f37d9dfa640 -1 osd.3 0 waiting for initial osdmap 2024-09-13T14:10:18.952 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:10:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[61509]: 2024-09-13T14:10:18.632+0000 7f37d6434640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:10:19.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:19 smithi189 ceph-mon[33294]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:10:19.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:19 smithi189 ceph-mon[33294]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:10:19.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:19 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:19.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:19 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:19.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[29399]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:10:19.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[29399]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:10:19.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:19.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:19.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[38255]: from='osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-13T14:10:19.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[38255]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:10:19.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:19.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:19 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:20.357 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:10:20.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi189:vg_nvme/lv_4 2024-09-13T14:10:20.658 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:10:20.904 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:10:20.904 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:10:20.904 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127] boot 2024-09-13T14:10:20.904 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:10:20.904 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:20 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:10:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:10:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127] boot 2024-09-13T14:10:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: osd.3 [v2:172.21.15.17:6826/2937054127,v1:172.21.15.17:6827/2937054127] boot 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:20.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:20 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:21.944 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:21 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi017 2024-09-13T14:10:21.944 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:21 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-13T14:10:21.944 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:21 smithi189 ceph-mon[33294]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi017 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[29399]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi017 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-13T14:10:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:21 smithi017 ceph-mon[38255]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:10:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:22 smithi189 ceph-mon[33294]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:22 smithi189 ceph-mon[33294]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:22.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:22 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:22.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:22 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:22.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:22 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[38255]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[38255]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[29399]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[29399]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:22 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/2281233142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]': finished 2024-09-13T14:10:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:10:25.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:25.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:24 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/1716788663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/2281233142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]': finished 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/1716788663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/2281233142' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]: dispatch 2024-09-13T14:10:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "698eddb3-30d9-431e-9731-9c37441f4359"}]': finished 2024-09-13T14:10:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:10:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:24 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/1716788663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:26 smithi189 ceph-mon[33294]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:26 smithi017 ceph-mon[29399]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:26 smithi017 ceph-mon[38255]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:29.141 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:28 smithi189 ceph-mon[33294]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:28 smithi017 ceph-mon[29399]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:28 smithi017 ceph-mon[38255]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:30 smithi189 ceph-mon[33294]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:10:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:30 smithi189 ceph-mon[33294]: Deploying daemon osd.4 on smithi189 2024-09-13T14:10:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[29399]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:10:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[29399]: Deploying daemon osd.4 on smithi189 2024-09-13T14:10:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[38255]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:10:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:30 smithi017 ceph-mon[38255]: Deploying daemon osd.4 on smithi189 2024-09-13T14:10:33.177 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:32 smithi189 ceph-mon[33294]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:33.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:32 smithi017 ceph-mon[38255]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:33.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:32 smithi017 ceph-mon[29399]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:34 smithi189 ceph-mon[33294]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:34 smithi017 ceph-mon[29399]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:34 smithi017 ceph-mon[38255]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:36.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:36.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:36.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:37.093 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:36 smithi189 ceph-mon[33294]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:36 smithi017 ceph-mon[29399]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:36 smithi017 ceph-mon[38255]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:38.587 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 4 on host 'smithi189' 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:39 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.185 DEBUG:teuthology.orchestra.run.smithi189:osd.4> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T14:10:39.187 INFO:tasks.cephadm:Deploying osd.5 on smithi189 with /dev/vg_nvme/lv_3... 2024-09-13T14:10:39.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:39 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:39.917 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:10:40.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-mon[33294]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:10:40.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-mon[33294]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:10:40.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:40.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-mon[33294]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:40.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-mon[33294]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:40.340 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T14:10:40.055+0000 7ff8792ff640 -1 osd.4 0 waiting for initial osdmap 2024-09-13T14:10:40.340 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 14:10:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T14:10:40.060+0000 7ff875939640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:10:40.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[29399]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:10:40.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[29399]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:10:40.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:40.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[29399]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:40.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[29399]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:40.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[38255]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:10:40.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[38255]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:10:40.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:40.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[38255]: from='osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:40.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:40 smithi017 ceph-mon[38255]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:10:41.445 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:41 smithi189 ceph-mon[33294]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:41.445 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:41 smithi189 ceph-mon[33294]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:10:41.445 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:41 smithi189 ceph-mon[33294]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:10:41.445 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:41.446 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:41 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[29399]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[29399]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[29399]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[38255]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:10:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[38255]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:10:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[38255]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:10:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:41 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:42.344 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:10:42.345 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:10:42.345 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996] boot 2024-09-13T14:10:42.345 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:10:42.345 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:42.345 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:42 smithi189 ceph-mon[33294]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996] boot 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[29399]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: osd.4 [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996] boot 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:10:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:42 smithi017 ceph-mon[38255]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:10:43.320 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:43 smithi189 ceph-mon[33294]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:43.320 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.320 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:43 smithi189 ceph-mon[33294]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:10:43.320 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.321 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[29399]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[29399]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:10:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[38255]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[38255]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:10:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi189 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 17088M 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi189 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 17088M 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:44 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi189 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 17088M 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:44.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:44.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:10:44.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:44 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:10:44.456 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:10:44.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi189:vg_nvme/lv_3 2024-09-13T14:10:44.719 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:10:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:45 smithi017 ceph-mon[29399]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:45 smithi017 ceph-mon[38255]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:45.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:45 smithi189 ceph-mon[33294]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[29399]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[38255]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:47 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:47 smithi189 ceph-mon[33294]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:47 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:10:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:47 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:10:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:47 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:48 smithi017 ceph-mon[29399]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:48 smithi017 ceph-mon[38255]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:48.476 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:48 smithi189 ceph-mon[33294]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:10:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/3752282899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]': finished 2024-09-13T14:10:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:10:49.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:49 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:10:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:10:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/3752282899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]': finished 2024-09-13T14:10:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:10:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:10:49.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-13T14:10:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/3752282899' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]: dispatch 2024-09-13T14:10:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "60f030c1-c222-4a1e-b7d6-d788f37bc5eb"}]': finished 2024-09-13T14:10:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:10:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:49 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:10:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:50 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/3958565716' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:50 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/3958565716' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:50 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/3958565716' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:10:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:51 smithi017 ceph-mon[29399]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:10:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:51 smithi017 ceph-mon[38255]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:10:51.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:51 smithi189 ceph-mon[33294]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:10:53.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:53 smithi189 ceph-mon[33294]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:10:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:53 smithi017 ceph-mon[29399]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:10:53.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:53 smithi017 ceph-mon[38255]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:10:55.212 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:55 smithi189 ceph-mon[33294]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:55.212 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:10:55.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[29399]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:10:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[38255]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:10:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:56 smithi017 ceph-mon[29399]: Deploying daemon osd.5 on smithi189 2024-09-13T14:10:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:56 smithi017 ceph-mon[38255]: Deploying daemon osd.5 on smithi189 2024-09-13T14:10:56.469 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:56 smithi189 ceph-mon[33294]: Deploying daemon osd.5 on smithi189 2024-09-13T14:10:57.300 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:57 smithi189 ceph-mon[33294]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:57 smithi017 ceph-mon[29399]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:57 smithi017 ceph-mon[38255]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:10:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:10:59 smithi017 ceph-mon[29399]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:10:59 smithi017 ceph-mon[38255]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:10:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:10:59 smithi189 ceph-mon[33294]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:00.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:00.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:00 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:00.499 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:00 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:00.499 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:00 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:00.499 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:00 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:01 smithi017 ceph-mon[38255]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:01 smithi017 ceph-mon[29399]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:01.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:01 smithi189 ceph-mon[33294]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:02.707 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 5 on host 'smithi189' 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[38255]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:02 smithi017 ceph-mon[29399]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:02 smithi189 ceph-mon[33294]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:11:03.321 DEBUG:teuthology.orchestra.run.smithi189:osd.5> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:11:03.323 INFO:tasks.cephadm:Deploying osd.6 on smithi189 with /dev/vg_nvme/lv_2... 2024-09-13T14:11:03.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:11:03.825 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:11:03.825 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.826 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.826 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[44832]: 2024-09-13T14:11:03.698+0000 7f88c5225640 -1 osd.5 0 waiting for initial osdmap 2024-09-13T14:11:03.826 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:11:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[44832]: 2024-09-13T14:11:03.704+0000 7f88c185f640 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:03 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:04.116 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:04.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:04 smithi189 ceph-mon[33294]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:04.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:04 smithi189 ceph-mon[33294]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:11:04.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:04 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:04.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:04 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:04.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:04 smithi189 ceph-mon[33294]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[29399]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[29399]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:11:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[29399]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[38255]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[38255]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:11:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:04 smithi017 ceph-mon[38255]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:11:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:11:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:11:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919] boot 2024-09-13T14:11:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:11:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:05 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919] boot 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:11:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: osd.5 [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919] boot 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:11:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:05 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:11:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:06 smithi189 ceph-mon[33294]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:11:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:06 smithi189 ceph-mon[33294]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:06 smithi189 ceph-mon[33294]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:11:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[29399]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:11:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[29399]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[29399]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:11:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[38255]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:11:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[38255]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:06 smithi017 ceph-mon[38255]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:11:08.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi189 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 8544M 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:08.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:08 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.413 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:11:08.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi189:vg_nvme/lv_2 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi189 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 8544M 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi189 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 8544M 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:08 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:08.673 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:09.372 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:09 smithi189 ceph-mon[33294]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:09 smithi017 ceph-mon[29399]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:09 smithi017 ceph-mon[38255]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:11 smithi189 ceph-mon[33294]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:11 smithi189 ceph-mon[33294]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:11 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:11 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:11 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[29399]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[29399]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[38255]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[38255]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:11 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/1705523809' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]': finished 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:11:13.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:13 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/1705523809' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]': finished 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:11:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/1705523809' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]: dispatch 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a53bc1f8-35e0-4e97-a9d1-90b318be2165"}]': finished 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:11:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:13 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:14 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/1415398999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:14.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:14 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/1415398999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:14 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/1415398999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:15.348 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:15 smithi189 ceph-mon[33294]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:15 smithi017 ceph-mon[29399]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:15 smithi017 ceph-mon[38255]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:17.238 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:17 smithi189 ceph-mon[33294]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:17 smithi017 ceph-mon[29399]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:17 smithi017 ceph-mon[38255]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:19.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:19 smithi189 ceph-mon[33294]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:19.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:19 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:11:19.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:19 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[29399]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:11:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[38255]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:11:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:19 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:20.180 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:20 smithi189 ceph-mon[33294]: Deploying daemon osd.6 on smithi189 2024-09-13T14:11:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:20 smithi017 ceph-mon[29399]: Deploying daemon osd.6 on smithi189 2024-09-13T14:11:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:20 smithi017 ceph-mon[38255]: Deploying daemon osd.6 on smithi189 2024-09-13T14:11:21.317 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:21 smithi189 ceph-mon[33294]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:21 smithi017 ceph-mon[29399]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:21 smithi017 ceph-mon[38255]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:23.268 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:23 smithi189 ceph-mon[33294]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:23 smithi017 ceph-mon[29399]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:23 smithi017 ceph-mon[38255]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:24 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:24 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:24 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:24 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:25.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:25 smithi189 ceph-mon[33294]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:25 smithi017 ceph-mon[38255]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:25 smithi017 ceph-mon[29399]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:26.751 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 6 on host 'smithi189' 2024-09-13T14:11:27.375 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.377 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.377 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:27.377 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.377 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:27 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.399 DEBUG:teuthology.orchestra.run.smithi189:osd.6> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T14:11:27.403 INFO:tasks.cephadm:Deploying osd.7 on smithi189 with /dev/vg_nvme/lv_1... 2024-09-13T14:11:27.403 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:11:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:11:27.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:27.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:27.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:27 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:28.170 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:28.397 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-mon[33294]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:11:28.397 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-mon[33294]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:11:28.397 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:28.398 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-mon[33294]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:28.398 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-mon[33294]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:28.398 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T14:11:28.120+0000 7fd5d0ac7640 -1 osd.6 0 waiting for initial osdmap 2024-09-13T14:11:28.398 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 14:11:28 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T14:11:28.126+0000 7fd5cd902640 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[29399]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[29399]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[29399]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[29399]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[38255]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:11:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[38255]: osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:11:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[38255]: from='osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:28 smithi017 ceph-mon[38255]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:29 smithi189 ceph-mon[33294]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:29 smithi189 ceph-mon[33294]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:29 smithi189 ceph-mon[33294]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:11:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:29 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:29 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[29399]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[29399]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[29399]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:11:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[38255]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[38255]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[38255]: osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:11:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:29 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256] boot 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:11:30.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:30 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:11:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:11:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256] boot 2024-09-13T14:11:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: osd.6 [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256] boot 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:11:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:30 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:11:31.387 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:31 smithi189 ceph-mon[33294]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:31.387 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:31 smithi189 ceph-mon[33294]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:11:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:31 smithi017 ceph-mon[29399]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:31 smithi017 ceph-mon[29399]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:11:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:31 smithi017 ceph-mon[38255]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:11:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:31 smithi017 ceph-mon[38255]: osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi189 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 5696M 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:32.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:32.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:32 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:11:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi189 2024-09-13T14:11:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 5696M 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi189 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 5696M 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:32.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:32 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:32.558 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-09-13T14:11:32.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi189:vg_nvme/lv_1 2024-09-13T14:11:32.821 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:11:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:33 smithi189 ceph-mon[33294]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:33 smithi017 ceph-mon[29399]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:33 smithi017 ceph-mon[38255]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:35 smithi189 ceph-mon[33294]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:35 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[29399]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[38255]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:11:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:11:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:35 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:36 smithi189 ceph-mon[33294]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:36 smithi017 ceph-mon[29399]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:36 smithi017 ceph-mon[38255]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/539993587' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]': finished 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:11:37.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:37 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/539993587' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]': finished 2024-09-13T14:11:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/539993587' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]: dispatch 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3fdb8ae-2c63-493d-b2ac-8009b024fa12"}]': finished 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:11:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:37 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:38 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/4272228779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:38 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/4272228779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:38 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/4272228779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:11:39.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:39 smithi189 ceph-mon[33294]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:39 smithi017 ceph-mon[29399]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:39 smithi017 ceph-mon[38255]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:41 smithi189 ceph-mon[33294]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:41.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:41 smithi017 ceph-mon[29399]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:41 smithi017 ceph-mon[38255]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:43.617 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:43 smithi189 ceph-mon[33294]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:43.618 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:11:43.618 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:43 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[29399]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:11:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[38255]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:11:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:43 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:44.379 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:44 smithi189 ceph-mon[33294]: Deploying daemon osd.7 on smithi189 2024-09-13T14:11:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:44 smithi017 ceph-mon[29399]: Deploying daemon osd.7 on smithi189 2024-09-13T14:11:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:44 smithi017 ceph-mon[38255]: Deploying daemon osd.7 on smithi189 2024-09-13T14:11:45.611 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:45 smithi189 ceph-mon[33294]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:45 smithi017 ceph-mon[29399]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:45 smithi017 ceph-mon[38255]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:47 smithi189 ceph-mon[33294]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:47 smithi017 ceph-mon[29399]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:47 smithi017 ceph-mon[38255]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:48.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:48.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:48.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:48 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:48 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:49 smithi017 ceph-mon[29399]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:49 smithi017 ceph-mon[38255]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:49.730 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:49 smithi189 ceph-mon[33294]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:51.307 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 7 on host 'smithi189' 2024-09-13T14:11:51.659 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:51 smithi189 ceph-mon[33294]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:51.659 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:51 smithi189 ceph-mon[33294]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.659 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:51 smithi189 ceph-mon[33294]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[29399]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[29399]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[29399]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[38255]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[38255]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:51 smithi017 ceph-mon[38255]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:11:51.958 DEBUG:teuthology.orchestra.run.smithi189:osd.7> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T14:11:51.961 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T14:11:51.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:11:52.229 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:52 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.755 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T14:11:52.289+0000 7f78b19fd640 -1 osd.7 0 waiting for initial osdmap 2024-09-13T14:11:52.756 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T14:11:52.296+0000 7f78ac834640 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:52.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:52 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107] boot 2024-09-13T14:11:53.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107] boot 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:11:53.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:53 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: osd.7 [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107] boot 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:11:53.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:53 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:11:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:11:54.720 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1726236712,"num_in_osds":8,"osd_in_since":1726236696,"num_remapped_pgs":0} 2024-09-13T14:11:54.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[38255]: purged_snaps scrub starts 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[38255]: purged_snaps scrub ok 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[38255]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[38255]: pgmap v164: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1425457890' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:11:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[29399]: purged_snaps scrub starts 2024-09-13T14:11:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[29399]: purged_snaps scrub ok 2024-09-13T14:11:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[29399]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:11:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[29399]: pgmap v164: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:54 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1425457890' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:11:54.980 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:55.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:54 smithi189 ceph-mon[33294]: purged_snaps scrub starts 2024-09-13T14:11:55.173 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:54 smithi189 ceph-mon[33294]: purged_snaps scrub ok 2024-09-13T14:11:55.173 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:54 smithi189 ceph-mon[33294]: osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:11:55.173 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:54 smithi189 ceph-mon[33294]: pgmap v164: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:55.173 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:54 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1425457890' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:11:56.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:55 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:11:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:56.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:55 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:11:56.681 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:11:56.681 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":51,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:11:54.758163+0000","last_up_change":"2024-09-13T14:11:52.754486+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6803","nonce":2866456241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6805","nonce":2866456241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6809","nonce":2866456241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6807","nonce":2866456241}]},"public_addr":"172.21.15.17:6803/2866456241","cluster_addr":"172.21.15.17:6805/2866456241","heartbeat_back_addr":"172.21.15.17:6809/2866456241","heartbeat_front_addr":"172.21.15.17:6807/2866456241","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6811","nonce":3915572123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6813","nonce":3915572123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6817","nonce":3915572123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6815","nonce":3915572123}]},"public_addr":"172.21.15.17:6811/3915572123","cluster_addr":"172.21.15.17:6813/3915572123","heartbeat_back_addr":"172.21.15.17:6817/3915572123","heartbeat_front_addr":"172.21.15.17:6815/3915572123","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6819","nonce":1775042177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6821","nonce":1775042177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6825","nonce":1775042177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6823","nonce":1775042177}]},"public_addr":"172.21.15.17:6819/1775042177","cluster_addr":"172.21.15.17:6821/1775042177","heartbeat_back_addr":"172.21.15.17:6825/1775042177","heartbeat_front_addr":"172.21.15.17:6823/1775042177","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6827","nonce":2937054127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6829","nonce":2937054127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6833","nonce":2937054127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6831","nonce":2937054127}]},"public_addr":"172.21.15.17:6827/2937054127","cluster_addr":"172.21.15.17:6829/2937054127","heartbeat_back_addr":"172.21.15.17:6833/2937054127","heartbeat_front_addr":"172.21.15.17:6831/2937054127","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[29399]: Detected new or changed devices on smithi189 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 4272M 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[29399]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2853929234' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[38255]: Detected new or changed devices on smithi189 2024-09-13T14:11:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 4272M 2024-09-13T14:11:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[38255]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:56 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2853929234' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:11:57.159 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:56 smithi189 ceph-mon[33294]: Detected new or changed devices on smithi189 2024-09-13T14:11:57.159 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:56 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 4272M 2024-09-13T14:11:57.159 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:56 smithi189 ceph-mon[33294]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:57.159 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:56 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2853929234' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:11:57.358 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T14:09:56.376395+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T14:11:57.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T14:11:57.614 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:58 smithi017 ceph-mon[29399]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:58 smithi017 ceph-mon[38255]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:59.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:58 smithi189 ceph-mon[33294]: pgmap v167: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:11:59.315 INFO:teuthology.orchestra.run.smithi017.stdout:pg_num: 1 2024-09-13T14:11:59.886 INFO:tasks.cephadm:Adding prometheus.a on smithi189 2024-09-13T14:11:59.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi189=a' 2024-09-13T14:12:00.134 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:11:59 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1300288111' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:12:00.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:11:59 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1300288111' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:12:00.240 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:11:59 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1300288111' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:12:01.221 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:00 smithi189 ceph-mon[33294]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-13T14:12:01.221 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:00 smithi189 ceph-mon[33294]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:12:01.221 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:00 smithi189 ceph-mon[33294]: Cluster is now healthy 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[29399]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[29399]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[29399]: Cluster is now healthy 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[38255]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[38255]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:12:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:00 smithi017 ceph-mon[38255]: Cluster is now healthy 2024-09-13T14:12:01.938 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled prometheus update... 2024-09-13T14:12:02.563 DEBUG:teuthology.orchestra.run.smithi189:prometheus.a> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T14:12:02.565 INFO:tasks.cephadm:Adding node-exporter.a on smithi017 2024-09-13T14:12:02.565 INFO:tasks.cephadm:Adding node-exporter.b on smithi189 2024-09-13T14:12:02.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi017=a;smithi189=b' 2024-09-13T14:12:02.876 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:12:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: Saving service prometheus spec with placement smithi189=a;count:1 2024-09-13T14:12:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[29399]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: Saving service prometheus spec with placement smithi189=a;count:1 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:03.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:02 smithi017 ceph-mon[38255]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: Saving service prometheus spec with placement smithi189=a;count:1 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:03.231 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:02 smithi189 ceph-mon[33294]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:12:05.188 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:04 smithi189 ceph-mon[33294]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:12:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:04 smithi017 ceph-mon[29399]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:12:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:04 smithi017 ceph-mon[38255]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:12:05.479 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled node-exporter update... 2024-09-13T14:12:06.402 DEBUG:teuthology.orchestra.run.smithi017:node-exporter.a> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T14:12:06.405 DEBUG:teuthology.orchestra.run.smithi189:node-exporter.b> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T14:12:06.408 INFO:tasks.cephadm:Adding alertmanager.a on smithi017 2024-09-13T14:12:06.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi017=a' 2024-09-13T14:12:06.689 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[29399]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[29399]: Saving service node-exporter spec with placement smithi017=a;smithi189=b;count:2 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[29399]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[38255]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[38255]: Saving service node-exporter spec with placement smithi017=a;smithi189=b;count:2 2024-09-13T14:12:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:06 smithi017 ceph-mon[38255]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T14:12:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:06 smithi189 ceph-mon[33294]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:06 smithi189 ceph-mon[33294]: Saving service node-exporter spec with placement smithi017=a;smithi189=b;count:2 2024-09-13T14:12:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:06 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:06 smithi189 ceph-mon[33294]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T14:12:09.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:08 smithi189 ceph-mon[33294]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:08 smithi017 ceph-mon[29399]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:08 smithi017 ceph-mon[38255]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:09.910 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled alertmanager update... 2024-09-13T14:12:11.093 DEBUG:teuthology.orchestra.run.smithi017:alertmanager.a> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T14:12:11.096 INFO:tasks.cephadm:Adding grafana.a on smithi189 2024-09-13T14:12:11.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi189=a' 2024-09-13T14:12:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[38255]: from='client.14397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[38255]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-13T14:12:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[38255]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[29399]: from='client.14397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[29399]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-13T14:12:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:10 smithi017 ceph-mon[29399]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:10 smithi189 ceph-mon[33294]: from='client.14397 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:10 smithi189 ceph-mon[33294]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-13T14:12:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:10 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:10 smithi189 ceph-mon[33294]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:12:11.941 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:12 smithi189 ceph-mon[33294]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:12 smithi017 ceph-mon[29399]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:12 smithi017 ceph-mon[38255]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:15.233 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:14 smithi189 ceph-mon[33294]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:14 smithi017 ceph-mon[29399]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:14 smithi017 ceph-mon[38255]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:15.763 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled grafana update... 2024-09-13T14:12:16.471 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:16 smithi189 systemd[1]: Starting Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:12:16.909 DEBUG:teuthology.orchestra.run.smithi189:grafana.a> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T14:12:16.912 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T14:12:16.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:12:17.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:16 smithi189 ceph-mon[33294]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:17.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:16 smithi189 ceph-mon[33294]: Saving service grafana spec with placement smithi189=a;count:1 2024-09-13T14:12:17.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:16 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:17.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:16 smithi189 ceph-mon[33294]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:17.177 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[29399]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[29399]: Saving service grafana spec with placement smithi189=a;count:1 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[29399]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[38255]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[38255]: Saving service grafana spec with placement smithi189=a;count:1 2024-09-13T14:12:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:16 smithi017 ceph-mon[38255]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:17.397 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 podman[62327]: 2024-09-13 14:12:17.013947804 +0000 UTC m=+0.016348747 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:12:17.398 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 podman[62327]: 2024-09-13 14:12:17.119414793 +0000 UTC m=+0.121815741 container create e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:17.398 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 podman[62327]: 2024-09-13 14:12:17.396983981 +0000 UTC m=+0.399384914 container init e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 podman[62327]: 2024-09-13 14:12:17.402589126 +0000 UTC m=+0.404990059 container start e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.426Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.427Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.427Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi189 (none))" 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.427Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.427Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.429Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.429Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.430Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:12:17.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.430Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.431Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.431Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.57µs 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.431Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.432Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.432Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.458µs wal_replay_duration=329.305µs wbl_replay_duration=100ns total_replay_duration=365.033µs 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.432Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.432Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.432Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.448Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.36144ms db_storage=764ns remote_storage=1.507µs web_handler=354ns query_engine=668ns scrape=382.791µs scrape_sd=15.246µs notify=700ns notify_sd=1.299µs rules=14.769085ms tracing=5.233µs 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.448Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:12:17.448Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:12:17.757 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 bash[62327]: e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd 2024-09-13T14:12:17.758 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:12:17 smithi189 systemd[1]: Started Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:18.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[29399]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:12:18.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mon[38255]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:18.824 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.824 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.824 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' 2024-09-13T14:12:18.824 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mon[33294]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ignoring --setuser ceph since I am not root 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ignoring --setgroup ceph since I am not root 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-mgr[35358]: -- 172.21.15.189:0/1349167208 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555ce445c340 con 0x555ce4468800 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:18.704+0000 7fa2d10d2ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:18.758+0000 7fa2d10d2ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:12:18.825 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:18.822+0000 7fa2d10d2ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setuser ceph since I am not root 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setgroup ceph since I am not root 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-mgr[29672]: -- 172.21.15.17:0/4115269159 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618c41da340 con 0x5618c41e6800 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:18.712+0000 7f98c22ccec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:18.771+0000 7f98c22ccec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:12:18.846 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:18.843+0000 7f98c22ccec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:12:19.061 INFO:teuthology.orchestra.run.smithi017.stdout:[client.0] 2024-09-13T14:12:19.061 INFO:teuthology.orchestra.run.smithi017.stdout: key = AQBDSORmQwJjAxAARBcyguEKn/OwlFb1O7JUtQ== 2024-09-13T14:12:19.083 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:18 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:18.946+0000 7fa2d10d2ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:12:19.126 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:18.974+0000 7f98c22ccec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:12:19.126 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:19.123+0000 7f98c22ccec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:12:19.338 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:19.082+0000 7fa2d10d2ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:12:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[29399]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:12:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[29399]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:12:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:19.400+0000 7f98c22ccec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:19.463+0000 7f98c22ccec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:19.636+0000 7f98c22ccec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[38255]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[38255]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:19 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-mon[33294]: from='mgr.14148 172.21.15.17:0/4220435655' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:12:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-mon[33294]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:12:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3288912160' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:19.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:19.337+0000 7fa2d10d2ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:12:19.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:19.398+0000 7fa2d10d2ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:12:19.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:19.559+0000 7fa2d10d2ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:12:19.778 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:12:19.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T14:12:19.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T14:12:19.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:12:20.071 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:20.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:20.445+0000 7fa2d10d2ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:12:20.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:20.514+0000 7fa2d10d2ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:12:20.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:20 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:20.538+0000 7f98c22ccec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:12:20.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:20 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:20.606+0000 7f98c22ccec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:12:21.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.232+0000 7fa2d10d2ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:12:21.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.352+0000 7fa2d10d2ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:12:21.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.474+0000 7fa2d10d2ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:12:21.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.365+0000 7f98c22ccec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:12:21.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.484+0000 7f98c22ccec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:12:21.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.611+0000 7f98c22ccec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:12:21.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.674+0000 7f98c22ccec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:12:21.803 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.537+0000 7fa2d10d2ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:12:21.803 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.669+0000 7fa2d10d2ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:12:21.803 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.739+0000 7fa2d10d2ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:12:21.803 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.801+0000 7fa2d10d2ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:12:21.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.810+0000 7f98c22ccec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:12:21.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.882+0000 7f98c22ccec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:12:22.005 INFO:teuthology.orchestra.run.smithi189.stdout:[client.1] 2024-09-13T14:12:22.006 INFO:teuthology.orchestra.run.smithi189.stdout: key = AQBGSORmUbUTABAAVjbVeahcl7NFNclL/pVYNw== 2024-09-13T14:12:22.055 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:21 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:21.942+0000 7fa2d10d2ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:12:22.055 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:22.008+0000 7fa2d10d2ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:12:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[29399]: from='client.? 172.21.15.189:0/2756902135' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[29399]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[29399]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:22.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:21.946+0000 7f98c22ccec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:12:22.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.092+0000 7f98c22ccec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:12:22.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.162+0000 7f98c22ccec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:12:22.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[38255]: from='client.? 172.21.15.189:0/2756902135' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[38255]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-mon[38255]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:22.081+0000 7fa2d10d2ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:22.163+0000 7fa2d10d2ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:22.221+0000 7fa2d10d2ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:22.281+0000 7fa2d10d2ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-mon[33294]: from='client.? 172.21.15.189:0/2756902135' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-mon[33294]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:12:22.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:22 smithi189 ceph-mon[33294]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:12:22.653 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-09-13T14:12:22.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T14:12:22.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T14:12:22.692 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T14:12:22.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T14:12:22.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.233+0000 7f98c22ccec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:12:22.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.321+0000 7f98c22ccec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:12:22.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.383+0000 7f98c22ccec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:12:22.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:22.445+0000 7f98c22ccec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:12:22.956 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:23.234 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: Standby manager daemon x restarted 2024-09-13T14:12:23.234 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: Standby manager daemon x started 2024-09-13T14:12:23.234 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:12:23.234 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: Standby manager daemon x restarted 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:12:23.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:12:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: Standby manager daemon x restarted 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/757653451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:23.039+0000 7fa2d10d2ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:12:23.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: [13/Sep/2024:14:12:23] ENGINE Bus STARTING 2024-09-13T14:12:23.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: CherryPy Checker: 2024-09-13T14:12:23.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: The Application mounted at '' has an empty config. 2024-09-13T14:12:23.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: 2024-09-13T14:12:23.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: [13/Sep/2024:14:12:23] ENGINE Serving on http://:::9283 2024-09-13T14:12:23.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:23 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: [13/Sep/2024:14:12:23] ENGINE Bus STARTED 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:23.275+0000 7f98c22ccec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:23] ENGINE Bus STARTING 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: CherryPy Checker: 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: The Application mounted at '' has an empty config. 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:23.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:23] ENGINE Serving on http://:::9283 2024-09-13T14:12:23.559 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:23 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:23] ENGINE Bus STARTED 2024-09-13T14:12:24.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: Active manager daemon y restarted 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: Activating manager daemon y 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: mgrmap e18: y(active, starting, since 0.00719723s), standbys: x 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:24.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: Manager daemon y is now available 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:24.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: Active manager daemon y restarted 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: Activating manager daemon y 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: mgrmap e18: y(active, starting, since 0.00719723s), standbys: x 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:12:24.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: Manager daemon y is now available 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:24.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:12:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:12:24.422 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:12:24.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: Active manager daemon y restarted 2024-09-13T14:12:24.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: Activating manager daemon y 2024-09-13T14:12:24.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:12:24.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: mgrmap e18: y(active, starting, since 0.00719723s), standbys: x 2024-09-13T14:12:24.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:12:24.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: Manager daemon y is now available 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:12:24.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:12:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:25.016+0000 7f986508f640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:12:25] ENGINE Error in HTTPServer.serve 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: Traceback (most recent call last): 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._connections.run(self.expiration_interval) 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._run(expiration_interval) 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: s = self.context.wrap_socket( 2024-09-13T14:12:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: return self.sslsocket_class._create( 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self.do_handshake() 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: self._sslobj.do_handshake() 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:12:25.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:12:25.293 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'rbd' created 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[29399]: mgrmap e19: y(active, since 1.01073s), standbys: x 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[29399]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[38255]: mgrmap e19: y(active, since 1.01073s), standbys: x 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[38255]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:12:25.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:25 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:25 smithi189 ceph-mon[33294]: mgrmap e19: y(active, since 1.01073s), standbys: x 2024-09-13T14:12:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:25 smithi189 ceph-mon[33294]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:25 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:12:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:25 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T14:12:26.359 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:26.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: [13/Sep/2024:14:12:24] ENGINE Bus STARTING 2024-09-13T14:12:26.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: [13/Sep/2024:14:12:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: [13/Sep/2024:14:12:25] ENGINE Bus STARTED 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: [13/Sep/2024:14:12:25] ENGINE Error in HTTPServer.serve 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:12:26.575 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: [13/Sep/2024:14:12:24] ENGINE Bus STARTING 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: [13/Sep/2024:14:12:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: [13/Sep/2024:14:12:25] ENGINE Bus STARTED 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: [13/Sep/2024:14:12:25] ENGINE Error in HTTPServer.serve 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:12:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:12:26.578 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:26 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: [13/Sep/2024:14:12:24] ENGINE Bus STARTING 2024-09-13T14:12:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: [13/Sep/2024:14:12:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:12:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: [13/Sep/2024:14:12:25] ENGINE Bus STARTED 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: [13/Sep/2024:14:12:25] ENGINE Error in HTTPServer.serve 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: Traceback (most recent call last): 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._run(expiration_interval) 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: s = self.context.wrap_socket( 2024-09-13T14:12:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: return self.sslsocket_class._create( 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: self.do_handshake() 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._sslobj.do_handshake() 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3425667154' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:26.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:26 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 4016M 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 4016M 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:27 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:27.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:12:27.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 4016M 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:27 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:12:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:12:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:12:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:12:28.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:28 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:12:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:28.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:12:28.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:12:28.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:28 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:12:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[29399]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:12:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:12:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[29399]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:12:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[38255]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:12:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:12:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:29 smithi017 ceph-mon[38255]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:12:29.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:12:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:12:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-mon[33294]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:12:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3046806525' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:12:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:29 smithi189 ceph-mon[33294]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:12:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[29399]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[29399]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:12:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[29399]: Cluster is now healthy 2024-09-13T14:12:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[29399]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:12:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[38255]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[38255]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:12:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[38255]: Cluster is now healthy 2024-09-13T14:12:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:30 smithi017 ceph-mon[38255]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:12:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:30 smithi189 ceph-mon[33294]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:30 smithi189 ceph-mon[33294]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:12:30.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:30 smithi189 ceph-mon[33294]: Cluster is now healthy 2024-09-13T14:12:30.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:30 smithi189 ceph-mon[33294]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:12:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:32 smithi017 ceph-mon[29399]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:12:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:32 smithi017 ceph-mon[29399]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:12:32.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:32 smithi017 ceph-mon[38255]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:12:32.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:32 smithi017 ceph-mon[38255]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:12:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:32 smithi189 ceph-mon[33294]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:12:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:32 smithi189 ceph-mon[33294]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-13T14:12:32.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T14:12:34.169 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:34 smithi017 ceph-mon[29399]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:12:34.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:34 smithi017 ceph-mon[38255]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:12:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:34 smithi189 ceph-mon[33294]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-13T14:12:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:36 smithi017 ceph-mon[29399]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 329 B/s wr, 11 op/s 2024-09-13T14:12:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:36 smithi017 ceph-mon[38255]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 329 B/s wr, 11 op/s 2024-09-13T14:12:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:36 smithi189 ceph-mon[33294]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 329 B/s wr, 11 op/s 2024-09-13T14:12:37.274 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:12:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4183950496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:12:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4183950496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:12:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4183950496' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:12:37.921 INFO:teuthology.orchestra.run.smithi017.stdout:[{"version":18,"timestamp":"2024-09-13T14:12:26.362816+0000","name":"","changes":[{"name":"osd/host:smithi189/osd_memory_target","previous_value":"4479546880","new_value":"4211111424"}]}] 2024-09-13T14:12:37.921 INFO:tasks.ceph_manager:config epoch is 18 2024-09-13T14:12:37.921 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:12:37.921 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:12:37.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:12:38.200 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 podman[69005]: 2024-09-13 14:12:38.026639236 +0000 UTC m=+0.017488468 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T14:12:38.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[29399]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T14:12:38.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[29399]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:12:38.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:38.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:38.549 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[38255]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T14:12:38.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[38255]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:12:38.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:38.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:38.550 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 podman[69005]: 2024-09-13 14:12:38.229312493 +0000 UTC m=+0.220161714 container create fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:38.550 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 podman[69005]: 2024-09-13 14:12:38.548958414 +0000 UTC m=+0.539807636 container init fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:38.737 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:38 smithi189 ceph-mon[33294]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 255 B/s wr, 9 op/s 2024-09-13T14:12:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:38 smithi189 ceph-mon[33294]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:12:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 podman[69005]: 2024-09-13 14:12:38.554295941 +0000 UTC m=+0.545145163 container start fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.566Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.566Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.567Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.568Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.591Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.591Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:12:38.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.594Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:12:38.835 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:38.594Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:12:38.835 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 bash[69005]: fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 2024-09-13T14:12:38.835 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:38 smithi017 systemd[1]: Started Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:39.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:38] ENGINE Bus STOPPING 2024-09-13T14:12:39.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:12:39.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:38] ENGINE Bus STOPPED 2024-09-13T14:12:39.100 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:38] ENGINE Bus STARTING 2024-09-13T14:12:39.100 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:39] ENGINE Serving on http://:::9283 2024-09-13T14:12:39.100 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:12:39] ENGINE Bus STARTED 2024-09-13T14:12:39.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: Deploying daemon grafana.a on smithi189 2024-09-13T14:12:39.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[38255]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 249 B/s wr, 0 op/s 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: Deploying daemon grafana.a on smithi189 2024-09-13T14:12:39.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:39 smithi017 ceph-mon[29399]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 249 B/s wr, 0 op/s 2024-09-13T14:12:40.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:12:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:12:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: Deploying daemon grafana.a on smithi189 2024-09-13T14:12:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:39 smithi189 ceph-mon[33294]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 249 B/s wr, 0 op/s 2024-09-13T14:12:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:12:40.829 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:40 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:40.568Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000325427s 2024-09-13T14:12:41.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:40 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/478145216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:12:41.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:40 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/478145216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:12:41.197 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":21,"flags":0,"active_gid":14412,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":2742834835},{"type":"v1","addr":"172.21.15.17:6801","nonce":2742834835}]},"active_addr":"172.21.15.17:6801/2742834835","active_change":"2024-09-13T14:12:23.277867+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1515064414}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":101070012}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3476899836}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2755559292}]}]} 2024-09-13T14:12:41.206 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:12:41.206 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:12:41.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:12:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:40 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/478145216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:12:41.465 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:41 smithi017 ceph-mon[29399]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:41 smithi017 ceph-mon[38255]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:41 smithi189 ceph-mon[33294]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:12:43.324 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":59,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:12:37.763274+0000","last_up_change":"2024-09-13T14:11:52.754486+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6803","nonce":2866456241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6805","nonce":2866456241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6809","nonce":2866456241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6807","nonce":2866456241}]},"public_addr":"172.21.15.17:6803/2866456241","cluster_addr":"172.21.15.17:6805/2866456241","heartbeat_back_addr":"172.21.15.17:6809/2866456241","heartbeat_front_addr":"172.21.15.17:6807/2866456241","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6811","nonce":3915572123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6813","nonce":3915572123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6817","nonce":3915572123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6815","nonce":3915572123}]},"public_addr":"172.21.15.17:6811/3915572123","cluster_addr":"172.21.15.17:6813/3915572123","heartbeat_back_addr":"172.21.15.17:6817/3915572123","heartbeat_front_addr":"172.21.15.17:6815/3915572123","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6819","nonce":1775042177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6821","nonce":1775042177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6825","nonce":1775042177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6823","nonce":1775042177}]},"public_addr":"172.21.15.17:6819/1775042177","cluster_addr":"172.21.15.17:6821/1775042177","heartbeat_back_addr":"172.21.15.17:6825/1775042177","heartbeat_front_addr":"172.21.15.17:6823/1775042177","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6827","nonce":2937054127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6829","nonce":2937054127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6833","nonce":2937054127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6831","nonce":2937054127}]},"public_addr":"172.21.15.17:6827/2937054127","cluster_addr":"172.21.15.17:6829/2937054127","heartbeat_back_addr":"172.21.15.17:6833/2937054127","heartbeat_front_addr":"172.21.15.17:6831/2937054127","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:12:43.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:43 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2568187757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:43.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:43.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:43 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2568187757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:43.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:43 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2568187757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:43.977 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:12:43.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:12:44.234 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:44.666 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:44 smithi017 ceph-mon[29399]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:44.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:44 smithi017 ceph-mon[38255]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:44 smithi189 ceph-mon[33294]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:12:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:12:46.368 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":59,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:12:37.763274+0000","last_up_change":"2024-09-13T14:11:52.754486+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6803","nonce":2866456241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6805","nonce":2866456241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6809","nonce":2866456241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6807","nonce":2866456241}]},"public_addr":"172.21.15.17:6803/2866456241","cluster_addr":"172.21.15.17:6805/2866456241","heartbeat_back_addr":"172.21.15.17:6809/2866456241","heartbeat_front_addr":"172.21.15.17:6807/2866456241","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6811","nonce":3915572123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6813","nonce":3915572123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6817","nonce":3915572123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6815","nonce":3915572123}]},"public_addr":"172.21.15.17:6811/3915572123","cluster_addr":"172.21.15.17:6813/3915572123","heartbeat_back_addr":"172.21.15.17:6817/3915572123","heartbeat_front_addr":"172.21.15.17:6815/3915572123","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6819","nonce":1775042177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6821","nonce":1775042177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6825","nonce":1775042177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6823","nonce":1775042177}]},"public_addr":"172.21.15.17:6819/1775042177","cluster_addr":"172.21.15.17:6821/1775042177","heartbeat_back_addr":"172.21.15.17:6825/1775042177","heartbeat_front_addr":"172.21.15.17:6823/1775042177","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6827","nonce":2937054127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6829","nonce":2937054127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6833","nonce":2937054127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6831","nonce":2937054127}]},"public_addr":"172.21.15.17:6827/2937054127","cluster_addr":"172.21.15.17:6829/2937054127","heartbeat_back_addr":"172.21.15.17:6833/2937054127","heartbeat_front_addr":"172.21.15.17:6831/2937054127","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:12:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:46 smithi017 ceph-mon[29399]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:46 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1681452481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:46 smithi017 ceph-mon[38255]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:46 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1681452481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:46 smithi189 ceph-mon[33294]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:46 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1681452481' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:12:46.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:12:46.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:12:46.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:12:46.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:12:46.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:12:46.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:12:46.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:12:46.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:12:47.434 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.434 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.435 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.444 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.455 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.456 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.457 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:47.484 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:48.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:48 smithi017 ceph-mon[29399]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:48.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:48 smithi017 ceph-mon[38255]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:48 smithi189 ceph-mon[33294]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:48.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:12:48 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:12:48.571Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00328364s 2024-09-13T14:12:49.953 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:12:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:12:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:50 smithi017 ceph-mon[38255]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:50 smithi017 ceph-mon[29399]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:50 smithi189 ceph-mon[33294]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:50.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:50 smithi189 podman[64420]: 2024-09-13 14:12:50.454633707 +0000 UTC m=+0.014963846 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:12:51.618 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 podman[64420]: 2024-09-13 14:12:51.282508235 +0000 UTC m=+0.842838418 container create da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , distribution-scope=public, io.openshift.tags=base rhel8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, vcs-type=git, release=239.1651231664, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-13T14:12:51.618 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 podman[64420]: 2024-09-13 14:12:51.52658037 +0000 UTC m=+1.086910500 container init da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, name=ubi8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, description=Ceph Grafana Container, release=239.1651231664) 2024-09-13T14:12:51.618 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 podman[64420]: 2024-09-13 14:12:51.53232662 +0000 UTC m=+1.092656745 container start da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, description=Ceph Grafana Container, com.redhat.component=ubi8-container, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , version=8.5, architecture=x86_64, io.buildah.version=1.32.0) 2024-09-13T14:12:51.618 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 bash[64420]: da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 systemd[1]: Started Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695311156Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695507874Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695527881Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695535528Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695542063Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695548323Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695554102Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:12:51.870 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.6955602Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695566337Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695572292Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695577973Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695583548Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.69558939Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=settings t=2024-09-13T14:12:51.695595472Z level=info msg="App mode production" 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore t=2024-09-13T14:12:51.695641289Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore t=2024-09-13T14:12:51.695657667Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.696480237Z level=info msg="Starting DB migrations" 2024-09-13T14:12:51.871 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.696551991Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.697052984Z level=info msg="Executing migration" id="create user table" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.697502542Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.700678663Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.703481152Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.706106019Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T14:12:51.872 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.709030685Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.711778905Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.714645371Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.715082751Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.717900447Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.720595225Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.723525102Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.726364219Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.729000687Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.732159645Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.734915641Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T14:12:51.873 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.7376955Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.740501935Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.743766704Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.746527594Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.749334546Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.752120864Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.754678583Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.757365978Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.760088097Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.762712001Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.765554573Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T14:12:51.874 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.768153249Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.770919251Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.771395179Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.774085465Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.776709634Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.779235443Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.779653629Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.782146196Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.784859964Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.787602524Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.788364586Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.789335594Z level=info msg="Executing migration" id="create star table" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.790047577Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T14:12:51.875 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.790942669Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.791709809Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.792548459Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.793398556Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.794248819Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.795257925Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.795898901Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.796813553Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.797588927Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.798486205Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.798961733Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.799373772Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T14:12:51.876 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.800253157Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.801092595Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.80194051Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.802790906Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.803643177Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.804507917Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T14:12:51.877 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.805321709Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.806289021Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.806784997Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.80717935Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.807469779Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.808003373Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.80848077Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.808934351Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.809354014Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.809802434Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.810244845Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T14:12:51.878 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.810623858Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.81088269Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.811173589Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.811696188Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.812467331Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.813406174Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.81415695Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.815088256Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.815813395Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.816788351Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.81752576Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.818419237Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.819004565Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.819908479Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T14:12:51.879 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.820862096Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.8217186Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.822454382Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.823241996Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.823642998Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.824187828Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.825029813Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.825920574Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.826887795Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T14:12:51.880 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.827574428Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.828450571Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.829308225Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.832264125Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.832742597Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.835588967Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.838144353Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.841144933Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.844077654Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.846970808Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T14:12:51.881 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.849589499Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.852352826Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.855056117Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.858006607Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.860751796Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.863576734Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.866335901Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.869092621Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T14:12:51.882 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.871659433Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T14:12:52.122 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.875004877Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T14:12:52.122 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.877751636Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T14:12:52.122 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.880580006Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T14:12:52.122 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.883192444Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.885792693Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.888407433Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.891178053Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.893910541Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.896516345Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.896947183Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.899708102Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.902305385Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.905095061Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T14:12:52.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.905564214Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T14:12:52.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.908469673Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T14:12:52.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.910992449Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T14:12:52.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.911327663Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T14:12:52.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.913698353Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T14:12:52.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.914105646Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.916298728Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.917082901Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.919040685Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.919793184Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.920397845Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.921192204Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.921600801Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.922031434Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.922892745Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T14:12:52.125 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.923332667Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.924121136Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.924947866Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.925861793Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.926425103Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.927257692Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.928021922Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.928879442Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.929333597Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.929604633Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.930207754Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.931025996Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.931877832Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T14:12:52.126 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.932238185Z level=info msg="Executing migration" id="create session table" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.932688967Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.93351893Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.933857575Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.934253389Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.935124188Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.935944413Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.936233534Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.936499895Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.936768083Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.937181905Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.937439756Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.938044989Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T14:12:52.127 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.938863894Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.93944902Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.940036769Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.940858216Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.941274159Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.942128833Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.942561593Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.943467367Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.944254795Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.944688033Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.94506508Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.945480933Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T14:12:52.128 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.945866122Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.947252826Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.947626149Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.948048956Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.948362919Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.948690662Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.94906791Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.94967372Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.950289384Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.950934582Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.951554007Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.951967748Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.952248217Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T14:12:52.129 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.952508531Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.952891816Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.953310868Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.953666687Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.954055462Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.954457962Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.955110173Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.955750784Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.956066563Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.956479559Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.95684594Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.957480671Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.957748764Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.958169799Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T14:12:52.130 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.958580467Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T14:12:52.131 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.958878435Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T14:12:52.131 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.959296897Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.95972729Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.960701432Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.961443515Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.962379665Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.963077235Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.965907159Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.968662933Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.969101681Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.972070409Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.974741661Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.977397124Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T14:12:52.132 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.97781167Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.98070474Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.983505052Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.986298516Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.986638758Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.989534308Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.992406096Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.995175744Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.995885023Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:51.99877978Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.001389102Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.004221766Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.005006162Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.007766346Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T14:12:52.133 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.010614871Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.013173114Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.013569652Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.016298146Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.018979114Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.02176829Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.022236332Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.024909025Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.025327233Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.027744496Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.030315891Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.030736455Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.033441844Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.036130105Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T14:12:52.134 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.038741101Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.039073046Z level=info msg="Executing migration" id="create team table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.040086345Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.04077562Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.041758597Z level=info msg="Executing migration" id="create team member table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.042268892Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.043120476Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.043947806Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.044790829Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.045515434Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.04635779Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.047175154Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.048017655Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.048467929Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.049298807Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T14:12:52.135 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.050105829Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.050960292Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.051846687Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.052612914Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.053509994Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.054348319Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.055341536Z level=info msg="Executing migration" id="create tag table" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.056029823Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.057036925Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.057755647Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.058706081Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.059576612Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T14:12:52.136 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.061636467Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.062637206Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.063349736Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.06434752Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.06505445Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.065935627Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.066794981Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.067738205Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.068590118Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.071353646Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.074177889Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.077019152Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.079803762Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.082654481Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T14:12:52.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.085309097Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.088234098Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.090972897Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.093680535Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.096569016Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.099246085Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.101984283Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.104734483Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.10772943Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.110368438Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.112957766Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.11582609Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.118708241Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T14:12:52.138 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.121449954Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T14:12:52.139 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.123986747Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T14:12:52.139 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.126704527Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.129319728Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.132179056Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.134936998Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.137411705Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.140153846Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.142728952Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.145410425Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.147892285Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.150606934Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T14:12:52.373 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.153274844Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.155814832Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.158683667Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.161321813Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.164001287Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.164998393Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.165763434Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T14:12:52.374 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.166628647Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.167444974Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.168213565Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.17149498Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.174776799Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.175748792Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.176493826Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.177376829Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.178232482Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T14:12:52.375 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.179043328Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.180041003Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.180753159Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.181731358Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.182664985Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.183516226Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.184368581Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.184798662Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.185297883Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.186137541Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.186965432Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.187406038Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.188251082Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T14:12:52.376 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.189099276Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.189403503Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.189850209Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.192606748Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.193178278Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.193481918Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.196342639Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.199788751Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.202518381Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.20533002Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.207956786Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.210711377Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.213200124Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.215937222Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T14:12:52.377 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.218756459Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.221497641Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.224145386Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.226723272Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.229507173Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.232180891Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.235050038Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.237665387Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.240495794Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.243015166Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.245724093Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.24861134Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.251126832Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T14:12:52.378 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.251573096Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.251645639Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.25246592Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.25299607Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.253428461Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.253559079Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.253673016Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.254305385Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.255252932Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.255974341Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.256835147Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T14:12:52.379 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.257659551Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.258654569Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.259349177Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.260317732Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.261042931Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.261925193Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.266246154Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.267253102Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.268073109Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.2724841Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.276749579Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.277601621Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T14:12:52.380 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.278442719Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.279400093Z level=info msg="Executing migration" id="create permission table" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.280184741Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.281005566Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.281875614Z level=info msg="Executing migration" id="create role table" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.282836369Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.28385081Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.284812278Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.285680518Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.286717188Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.287608981Z level=info msg="Executing migration" id="create team role table" 2024-09-13T14:12:52.381 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.288332367Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.289266551Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.290001165Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.290875688Z level=info msg="Executing migration" id="create user role table" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.291815147Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.294660324Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.297434293Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.30007353Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.302943687Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.305841705Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T14:12:52.382 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.30854954Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.311438149Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.312016679Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.314797679Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.315301827Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.315732305Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.316147739Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.316597633Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T14:12:52.383 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.317572594Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.317996071Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.318443176Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.318723813Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.319208675Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.319653336Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.320036548Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.32035165Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.3206941Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.321110592Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.32155537Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.322520125Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.322836368Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.323323987Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T14:12:52.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.323771584Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.324272262Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.325308882Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.325768017Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.326235695Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.326325817Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.326667087Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.326758874Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.327056488Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.327486983Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.327874988Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.328308129Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.32873199Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:12:52.385 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.329172889Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.329588164Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.330056418Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.330469794Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.330874869Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.331323788Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.334198121Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.337259972Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.338341013Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.339399909Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.339719304Z level=info msg="Executing migration" id="add share column" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.340728697Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.341192861Z level=info msg="Executing migration" id="create file table" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.341620512Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T14:12:52.386 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.344438473Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.344941608Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.347878029Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.350515875Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.35321434Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.356015755Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.358973135Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.361652377Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.364325551Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.367064893Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.369783702Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.372429131Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T14:12:52.387 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.375185006Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T14:12:52.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[29399]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:52.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[38255]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:52.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:52 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-mon[33294]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.377843575Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.378872015Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.379890909Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.380689508Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.390693899Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.391534412Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T14:12:52.756 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.392374754Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.395975938Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.396919355Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.397655573Z level=info msg="Executing migration" id="create folder table" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.398501639Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.399347029Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.400198631Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.401057538Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=migrator t=2024-09-13T14:12:52.401911147Z level=info msg="migrations completed" performed=485 skipped=0 duration=705.377876ms 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore t=2024-09-13T14:12:52.402317872Z level=info msg="Created default organization" 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=plugin.loader t=2024-09-13T14:12:52.459625527Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=plugin.loader t=2024-09-13T14:12:52.486636054Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=plugin.loader t=2024-09-13T14:12:52.486653656Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=secrets t=2024-09-13T14:12:52.486687946Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:12:52.757 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=query_data t=2024-09-13T14:12:52.488437786Z level=info msg="Query Service initialization" 2024-09-13T14:12:52.758 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=live.push_http t=2024-09-13T14:12:52.490707694Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:12:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-mon[33294]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:12:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=infra.usagestats.collector t=2024-09-13T14:12:53.227588268Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=provisioning.datasources t=2024-09-13T14:12:53.227919766Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=provisioning.datasources t=2024-09-13T14:12:53.260806529Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=provisioning.datasources t=2024-09-13T14:12:53.29229311Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=provisioning.alerting t=2024-09-13T14:12:53.314048652Z level=info msg="starting to provision alerting" 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=provisioning.alerting t=2024-09-13T14:12:53.314069131Z level=info msg="finished to provision alerting" 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=ngalert.state.manager t=2024-09-13T14:12:53.314358106Z level=info msg="Warming state cache for startup" 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=grafanaStorageLogger t=2024-09-13T14:12:53.314895522Z level=info msg="storage starting" 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=http.server t=2024-09-13T14:12:53.320240434Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=ngalert.state.manager t=2024-09-13T14:12:53.324774405Z level=info msg="State cache has been initialized" states=0 duration=10.415243ms 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:12:53.324830653Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:12:53.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=ticker t=2024-09-13T14:12:53.324855939Z level=info msg=starting first_tick=2024-09-13T14:13:00Z 2024-09-13T14:12:53.507 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore.transactions t=2024-09-13T14:12:53.3694882Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:12:53.507 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore.transactions t=2024-09-13T14:12:53.393988039Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:12:53.507 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:12:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[64434]: logger=sqlstore.transactions t=2024-09-13T14:12:53.404376599Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[38255]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[29399]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:12:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:53 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:12:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:54 smithi017 ceph-mon[29399]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:54 smithi017 ceph-mon[38255]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:54 smithi189 ceph-mon[33294]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:56 smithi017 ceph-mon[29399]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:56 smithi017 ceph-mon[38255]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:56 smithi189 ceph-mon[33294]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:57.184 INFO:teuthology.orchestra.run.smithi017.stdout:184683593747 2024-09-13T14:12:57.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:12:57.596 INFO:teuthology.orchestra.run.smithi017.stdout:34359738414 2024-09-13T14:12:57.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:12:58.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:12:58 smithi017 ceph-mon[29399]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:58.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:12:58 smithi017 ceph-mon[38255]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:12:58 smithi189 ceph-mon[33294]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:12:59.824 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:12:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:12:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:00 smithi017 ceph-mon[29399]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:00 smithi017 ceph-mon[38255]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:00 smithi189 ceph-mon[33294]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:01.428 INFO:teuthology.orchestra.run.smithi017.stdout:158913789978 2024-09-13T14:13:01.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:02 smithi017 ceph-mon[29399]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:02.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:02 smithi017 ceph-mon[38255]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:02 smithi189 ceph-mon[33294]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:03.710 INFO:teuthology.orchestra.run.smithi017.stdout:210453397520 2024-09-13T14:13:03.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:13:04.188 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:04 smithi017 podman[71171]: 2024-09-13 14:13:04.070235311 +0000 UTC m=+5.218040089 container create 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:04.515 INFO:teuthology.orchestra.run.smithi017.stdout:55834574892 2024-09-13T14:13:04.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:13:04.559 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:04 smithi017 ceph-mon[29399]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:04.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:04 smithi017 ceph-mon[38255]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:04 smithi189 ceph-mon[33294]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:05.429 INFO:teuthology.orchestra.run.smithi017.stdout:77309411368 2024-09-13T14:13:05.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:13:06.014 INFO:teuthology.orchestra.run.smithi017.stdout:133143986206 2024-09-13T14:13:06.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:13:06.351 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 podman[71171]: 2024-09-13 14:13:06.149268944 +0000 UTC m=+7.297073739 container init 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:06.351 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 podman[71171]: 2024-09-13 14:13:06.155254707 +0000 UTC m=+7.303059486 container start 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:06.351 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.159Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.159Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:13:06.352 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:13:06.353 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.160Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:13:06.354 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:13:06.355 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[71649]: ts=2024-09-13T14:13:06.161Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 bash[71171]: 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 2024-09-13T14:13:06.356 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:13:06 smithi017 systemd[1]: Started Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[29399]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[38255]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:06 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:06 smithi189 ceph-mon[33294]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:06 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:06 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:06 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:07.099 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:07.103 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:07.135 INFO:teuthology.orchestra.run.smithi017.stdout:111669149730 2024-09-13T14:13:07.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:13:07.138 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:07.166 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:06 smithi189 systemd[1]: Starting Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:13:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:07 smithi189 ceph-mon[33294]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:13:07.505 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:07 smithi189 bash[64713]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:13:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:07 smithi017 ceph-mon[29399]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:13:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:07 smithi017 ceph-mon[38255]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:13:08.005 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:07 smithi189 bash[64713]: Getting image source signatures 2024-09-13T14:13:08.005 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:07 smithi189 bash[64713]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:13:08.005 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:07 smithi189 bash[64713]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:13:08.005 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:07 smithi189 bash[64713]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:13:08.106 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:08.106 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:08.107 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:08.107 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:08.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:08 smithi017 ceph-mon[29399]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:08.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:08.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:08 smithi017 ceph-mon[38255]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:08.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:08 smithi189 ceph-mon[33294]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:09.255 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:08 smithi189 bash[64713]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:13:09.255 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 bash[64713]: Writing manifest to image destination 2024-09-13T14:13:09.571 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:09.571 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 podman[64713]: 2024-09-13 14:13:09.228607511 +0000 UTC m=+2.154064562 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:13:09.572 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 podman[64713]: 2024-09-13 14:13:09.344169885 +0000 UTC m=+2.269626923 container create b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:13:09.572 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 podman[64713]: 2024-09-13 14:13:09.571334874 +0000 UTC m=+2.496791912 container init b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:13:09.963 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 podman[64713]: 2024-09-13 14:13:09.576237026 +0000 UTC m=+2.501694067 container start b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.579Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.579Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.580Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.580Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:13:10.006 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.580Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.580Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:13:10.007 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:13:10.008 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:13:10.009 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:13:10.010 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:13:10.011 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[64787]: ts=2024-09-13T14:13:09.581Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:13:10.011 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 bash[64713]: b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 2024-09-13T14:13:10.011 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:13:09 smithi189 systemd[1]: Started Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:10.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:10.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:10 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:10 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:12.659 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:12 smithi189 ceph-mon[33294]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:12.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:12 smithi017 ceph-mon[29399]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:12.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:12 smithi017 ceph-mon[38255]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:14.256 INFO:teuthology.orchestra.run.smithi017.stdout:184683593750 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[38255]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/225212866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[29399]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:14.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:14 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/225212866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:13:14.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:14 smithi189 ceph-mon[33294]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:14.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:14 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:14 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:14 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:14.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:14 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/225212866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:13:16.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:16 smithi017 ceph-mon[29399]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:16.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:16 smithi017 ceph-mon[38255]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:16.715 INFO:teuthology.orchestra.run.smithi017.stdout:77309411370 2024-09-13T14:13:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:16 smithi189 ceph-mon[33294]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:17 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/389852997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:13:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:17 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/389852997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:13:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:17 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/389852997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:13:17.841 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411370 for osd.2 2024-09-13T14:13:17.842 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[29399]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1632827927' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2862583333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[38255]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1632827927' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:13:18.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:18 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2862583333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:13:18.486 INFO:teuthology.orchestra.run.smithi017.stdout:133143986209 2024-09-13T14:13:18.750 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593747 got 184683593750 for osd.6 2024-09-13T14:13:18.750 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:18 smithi189 ceph-mon[33294]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:18 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1632827927' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:13:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:18 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2862583333' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:13:18.977 INFO:teuthology.orchestra.run.smithi017.stdout:158913789981 2024-09-13T14:13:19.237 INFO:teuthology.orchestra.run.smithi017.stdout:210453397523 2024-09-13T14:13:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/156409566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:13:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:19 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/156409566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:13:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:19 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/156409566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:13:19.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:19.784 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397520 got 210453397523 for osd.7 2024-09-13T14:13:19.784 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:20.610 INFO:teuthology.orchestra.run.smithi017.stdout:111669149733 2024-09-13T14:13:20.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:20 smithi017 ceph-mon[29399]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:20.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:20 smithi017 ceph-mon[38255]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:20 smithi189 ceph-mon[33294]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:20.793 INFO:teuthology.orchestra.run.smithi017.stdout:55834574896 2024-09-13T14:13:21.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:21 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1721046716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:13:21.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:21 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1882549946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:13:21.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:21 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1721046716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:13:21.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:21 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1882549946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:13:21.443 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-09-13T14:13:21.443 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:21 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1721046716' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:13:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:21 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1882549946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:13:21.932 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986206 got 133143986209 for osd.4 2024-09-13T14:13:21.932 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:22.378 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149733 for osd.3 2024-09-13T14:13:22.378 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:22.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:22 smithi017 ceph-mon[29399]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:22.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:22 smithi017 ceph-mon[38255]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:22 smithi189 ceph-mon[33294]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:22.969 INFO:teuthology.orchestra.run.smithi017.stdout:34359738420 2024-09-13T14:13:23.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789978 got 158913789981 for osd.5 2024-09-13T14:13:23.195 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:23.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:23 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/879397301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:13:23.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:23.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:23 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/879397301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:13:23.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:23.688 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738420 for osd.0 2024-09-13T14:13:23.688 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:23.688 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:13:23.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:13:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:23 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/879397301' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:13:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:24.350 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:24.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:24 smithi017 ceph-mon[29399]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:24.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:24 smithi017 ceph-mon[38255]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:24 smithi189 ceph-mon[33294]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:26.210 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:26.211 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:13:26.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:26 smithi017 ceph-mon[29399]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:26.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:26 smithi017 ceph-mon[38255]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:26 smithi189 ceph-mon[33294]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:27.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:27 smithi017 ceph-mon[29399]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:27.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:27 smithi017 ceph-mon[38255]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:27.581 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-09-13T14:13:25.298849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385156,"kb_used_data":6308,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576060,"statfs":{"total":767960285184,"available":765517885440,"internally_reserved":0,"allocated":6459392,"data_stored":3738361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002108"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182982+0000","last_change":"2024-09-13T14:12:31.087292+0000","last_active":"2024-09-13T14:12:38.182982+0000","last_peered":"2024-09-13T14:12:38.182982+0000","last_clean":"2024-09-13T14:12:38.182982+0000","last_became_active":"2024-09-13T14:12:26.296590+0000","last_became_peered":"2024-09-13T14:12:26.296590+0000","last_unstale":"2024-09-13T14:12:38.182982+0000","last_undegraded":"2024-09-13T14:12:38.182982+0000","last_fullsized":"2024-09-13T14:12:38.182982+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:02.313429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026736199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.181854+0000","last_change":"2024-09-13T14:12:31.081944+0000","last_active":"2024-09-13T14:12:38.181854+0000","last_peered":"2024-09-13T14:12:38.181854+0000","last_clean":"2024-09-13T14:12:38.181854+0000","last_became_active":"2024-09-13T14:12:26.816862+0000","last_became_peered":"2024-09-13T14:12:26.816862+0000","last_unstale":"2024-09-13T14:12:38.181854+0000","last_undegraded":"2024-09-13T14:12:38.181854+0000","last_fullsized":"2024-09-13T14:12:38.181854+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:43.703026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062990400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182276+0000","last_change":"2024-09-13T14:12:31.172062+0000","last_active":"2024-09-13T14:12:38.182276+0000","last_peered":"2024-09-13T14:12:38.182276+0000","last_clean":"2024-09-13T14:12:38.182276+0000","last_became_active":"2024-09-13T14:12:26.816843+0000","last_became_peered":"2024-09-13T14:12:26.816843+0000","last_unstale":"2024-09-13T14:12:38.182276+0000","last_undegraded":"2024-09-13T14:12:38.182276+0000","last_fullsized":"2024-09-13T14:12:38.182276+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:53.145885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343452,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.181787+0000","last_change":"2024-09-13T14:12:31.081654+0000","last_active":"2024-09-13T14:12:38.181787+0000","last_peered":"2024-09-13T14:12:38.181787+0000","last_clean":"2024-09-13T14:12:38.181787+0000","last_became_active":"2024-09-13T14:12:26.551473+0000","last_became_peered":"2024-09-13T14:12:26.551473+0000","last_unstale":"2024-09-13T14:12:38.181787+0000","last_undegraded":"2024-09-13T14:12:38.181787+0000","last_fullsized":"2024-09-13T14:12:38.181787+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:42:25.018815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000346795,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182060+0000","last_change":"2024-09-13T14:12:31.086545+0000","last_active":"2024-09-13T14:12:38.182060+0000","last_peered":"2024-09-13T14:12:38.182060+0000","last_clean":"2024-09-13T14:12:38.182060+0000","last_became_active":"2024-09-13T14:12:26.297541+0000","last_became_peered":"2024-09-13T14:12:26.297541+0000","last_unstale":"2024-09-13T14:12:38.182060+0000","last_undegraded":"2024-09-13T14:12:38.182060+0000","last_fullsized":"2024-09-13T14:12:38.182060+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:54.129876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001036726,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:37.766038+0000","last_change":"2024-09-13T14:12:31.081764+0000","last_active":"2024-09-13T14:12:37.766038+0000","last_peered":"2024-09-13T14:12:37.766038+0000","last_clean":"2024-09-13T14:12:37.766038+0000","last_became_active":"2024-09-13T14:12:26.551472+0000","last_became_peered":"2024-09-13T14:12:26.551472+0000","last_unstale":"2024-09-13T14:12:37.766038+0000","last_undegraded":"2024-09-13T14:12:37.766038+0000","last_fullsized":"2024-09-13T14:12:37.766038+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:04:11.250981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039453399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182339+0000","last_change":"2024-09-13T14:12:31.171913+0000","last_active":"2024-09-13T14:12:38.182339+0000","last_peered":"2024-09-13T14:12:38.182339+0000","last_clean":"2024-09-13T14:12:38.182339+0000","last_became_active":"2024-09-13T14:12:26.551516+0000","last_became_peered":"2024-09-13T14:12:26.551516+0000","last_unstale":"2024-09-13T14:12:38.182339+0000","last_undegraded":"2024-09-13T14:12:38.182339+0000","last_fullsized":"2024-09-13T14:12:38.182339+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.648647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029229000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182212+0000","last_change":"2024-09-13T14:12:31.085929+0000","last_active":"2024-09-13T14:12:38.182212+0000","last_peered":"2024-09-13T14:12:38.182212+0000","last_clean":"2024-09-13T14:12:38.182212+0000","last_became_active":"2024-09-13T14:12:26.296739+0000","last_became_peered":"2024-09-13T14:12:26.296739+0000","last_unstale":"2024-09-13T14:12:38.182212+0000","last_undegraded":"2024-09-13T14:12:38.182212+0000","last_fullsized":"2024-09-13T14:12:38.182212+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:14.744825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042041700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"53'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:12:31.171615+0000","last_change":"2024-09-13T14:11:54.779008+0000","last_active":"2024-09-13T14:12:31.171615+0000","last_peered":"2024-09-13T14:12:31.171615+0000","last_clean":"2024-09-13T14:12:31.171615+0000","last_became_active":"2024-09-13T14:11:54.772279+0000","last_became_peered":"2024-09-13T14:11:54.772279+0000","last_unstale":"2024-09-13T14:12:31.171615+0000","last_undegraded":"2024-09-13T14:12:31.171615+0000","last_fullsized":"2024-09-13T14:12:31.171615+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:02.973115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397525,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.751},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.63400000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.68700000000000006}]},{"osd":2,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.65900000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.71299999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.432},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.56200000000000006}]},{"osd":5,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.624}]}]},{"osd":6,"up_from":43,"seq":184683593753,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.752}]},{"osd":1,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.78900000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.69399999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.70699999999999996}]},{"osd":4,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.66900000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.83199999999999996}]},{"osd":7,"last update":"Fri Sep 13 14:12:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.76800000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789983,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.67600000000000005,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.55700000000000005,"15min":0.56799999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.64000000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48199999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55900000000000005,"15min":0.56399999999999995},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69799999999999995,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55300000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.55900000000000005,"15min":0.56599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70199999999999996,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72399999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.56299999999999994,"15min":0.56799999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.629}]},{"osd":4,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39200000000000002,"15min":0.38900000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67700000000000005,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.67400000000000004}]},{"osd":7,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.76500000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.36099999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.34300000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.51700000000000002,"15min":0.52100000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63100000000000001,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.52900000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51500000000000001,"15min":0.51300000000000001},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.311},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.54500000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.51000000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.46800000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:13:09 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.33500000000000002},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.35799999999999998,"15min":0.34300000000000003},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.35899999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.41199999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:12:52 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.496},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.56000000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:24 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.28100000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57799999999999996,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.36599999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:12:24 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.433},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.53900000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.33400000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:12:24 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47099999999999997,"15min":0.46899999999999997},"min":{"1min":0.313,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.66000000000000003,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.46600000000000003,"15min":0.46500000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64200000000000002,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.40999999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:12:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84999999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.73799999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.68700000000000006}]},{"osd":6,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.86099999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.84899999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.374,"15min":0.36599999999999999},"min":{"1min":0.29799999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.371},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.29399999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.29399999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.376},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51300000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.41799999999999998,"15min":0.41299999999999998},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.627,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.313}]},{"osd":3,"last update":"Fri Sep 13 14:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.46200000000000002,"15min":0.46400000000000002},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.55100000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.44700000000000001,"15min":0.45300000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.53100000000000003,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.33600000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:12:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.60399999999999998,"15min":0.59399999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.67000000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:13:05 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.56899999999999995,"15min":0.56299999999999994},"min":{"1min":0.42199999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.46000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73499999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59199999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.52400000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.63500000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35699999999999998,"15min":0.34699999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57499999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40200000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38800000000000001,"15min":0.38700000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.53000000000000003,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.311},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40999999999999998,"15min":0.40500000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.32600000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:12:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.443,"15min":0.442},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45500000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.38100000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:12:42 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.878,"15min":0.878},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81699999999999995,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.47699999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:13:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59299999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.48699999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.45600000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.44600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":836141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:12:40 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.34999999999999998,"15min":0.33900000000000002},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.51500000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.316},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.35999999999999999,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35499999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:13:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38700000000000001,"15min":0.38300000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.40799999999999997},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.34300000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:13:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44400000000000001,"15min":0.44600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46000000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.372}]},{"osd":4,"last update":"Fri Sep 13 14:12:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.61599999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:13:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.625},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.53900000000000003}]},{"osd":7,"last update":"Fri Sep 13 14:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.71599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:13:27.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:13:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:27 smithi189 ceph-mon[33294]: from='client.24491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:28.370 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:28.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:28 smithi189 ceph-mon[33294]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:28.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:28.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:28 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:28.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[29399]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:28.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:28.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:28.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[38255]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:28.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:28.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:28 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:29.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[29399]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[29399]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[29399]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[38255]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[38255]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-13T14:13:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:30 smithi017 ceph-mon[38255]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:30 smithi189 ceph-mon[33294]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:13:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:30 smithi189 ceph-mon[33294]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-13T14:13:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:30 smithi189 ceph-mon[33294]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:30.997 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:30.998 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:13:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:32 smithi017 ceph-mon[29399]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:32 smithi017 ceph-mon[29399]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:32 smithi017 ceph-mon[38255]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:32 smithi017 ceph-mon[38255]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:32 smithi189 ceph-mon[33294]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:13:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:32 smithi189 ceph-mon[33294]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:32.770 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-13T14:13:29.299723+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385156,"kb_used_data":6308,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576060,"statfs":{"total":767960285184,"available":765517885440,"internally_reserved":0,"allocated":6459392,"data_stored":3738361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002056"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182982+0000","last_change":"2024-09-13T14:12:31.087292+0000","last_active":"2024-09-13T14:12:38.182982+0000","last_peered":"2024-09-13T14:12:38.182982+0000","last_clean":"2024-09-13T14:12:38.182982+0000","last_became_active":"2024-09-13T14:12:26.296590+0000","last_became_peered":"2024-09-13T14:12:26.296590+0000","last_unstale":"2024-09-13T14:12:38.182982+0000","last_undegraded":"2024-09-13T14:12:38.182982+0000","last_fullsized":"2024-09-13T14:12:38.182982+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:02.313429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026736199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.181854+0000","last_change":"2024-09-13T14:12:31.081944+0000","last_active":"2024-09-13T14:12:38.181854+0000","last_peered":"2024-09-13T14:12:38.181854+0000","last_clean":"2024-09-13T14:12:38.181854+0000","last_became_active":"2024-09-13T14:12:26.816862+0000","last_became_peered":"2024-09-13T14:12:26.816862+0000","last_unstale":"2024-09-13T14:12:38.181854+0000","last_undegraded":"2024-09-13T14:12:38.181854+0000","last_fullsized":"2024-09-13T14:12:38.181854+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:43.703026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062990400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182276+0000","last_change":"2024-09-13T14:12:31.172062+0000","last_active":"2024-09-13T14:12:38.182276+0000","last_peered":"2024-09-13T14:12:38.182276+0000","last_clean":"2024-09-13T14:12:38.182276+0000","last_became_active":"2024-09-13T14:12:26.816843+0000","last_became_peered":"2024-09-13T14:12:26.816843+0000","last_unstale":"2024-09-13T14:12:38.182276+0000","last_undegraded":"2024-09-13T14:12:38.182276+0000","last_fullsized":"2024-09-13T14:12:38.182276+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:53.145885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343452,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.181787+0000","last_change":"2024-09-13T14:12:31.081654+0000","last_active":"2024-09-13T14:12:38.181787+0000","last_peered":"2024-09-13T14:12:38.181787+0000","last_clean":"2024-09-13T14:12:38.181787+0000","last_became_active":"2024-09-13T14:12:26.551473+0000","last_became_peered":"2024-09-13T14:12:26.551473+0000","last_unstale":"2024-09-13T14:12:38.181787+0000","last_undegraded":"2024-09-13T14:12:38.181787+0000","last_fullsized":"2024-09-13T14:12:38.181787+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:42:25.018815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000346795,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182060+0000","last_change":"2024-09-13T14:12:31.086545+0000","last_active":"2024-09-13T14:12:38.182060+0000","last_peered":"2024-09-13T14:12:38.182060+0000","last_clean":"2024-09-13T14:12:38.182060+0000","last_became_active":"2024-09-13T14:12:26.297541+0000","last_became_peered":"2024-09-13T14:12:26.297541+0000","last_unstale":"2024-09-13T14:12:38.182060+0000","last_undegraded":"2024-09-13T14:12:38.182060+0000","last_fullsized":"2024-09-13T14:12:38.182060+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:54.129876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001036726,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:37.766038+0000","last_change":"2024-09-13T14:12:31.081764+0000","last_active":"2024-09-13T14:12:37.766038+0000","last_peered":"2024-09-13T14:12:37.766038+0000","last_clean":"2024-09-13T14:12:37.766038+0000","last_became_active":"2024-09-13T14:12:26.551472+0000","last_became_peered":"2024-09-13T14:12:26.551472+0000","last_unstale":"2024-09-13T14:12:37.766038+0000","last_undegraded":"2024-09-13T14:12:37.766038+0000","last_fullsized":"2024-09-13T14:12:37.766038+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:04:11.250981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039453399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182339+0000","last_change":"2024-09-13T14:12:31.171913+0000","last_active":"2024-09-13T14:12:38.182339+0000","last_peered":"2024-09-13T14:12:38.182339+0000","last_clean":"2024-09-13T14:12:38.182339+0000","last_became_active":"2024-09-13T14:12:26.551516+0000","last_became_peered":"2024-09-13T14:12:26.551516+0000","last_unstale":"2024-09-13T14:12:38.182339+0000","last_undegraded":"2024-09-13T14:12:38.182339+0000","last_fullsized":"2024-09-13T14:12:38.182339+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.648647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029229000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:12:38.182212+0000","last_change":"2024-09-13T14:12:31.085929+0000","last_active":"2024-09-13T14:12:38.182212+0000","last_peered":"2024-09-13T14:12:38.182212+0000","last_clean":"2024-09-13T14:12:38.182212+0000","last_became_active":"2024-09-13T14:12:26.296739+0000","last_became_peered":"2024-09-13T14:12:26.296739+0000","last_unstale":"2024-09-13T14:12:38.182212+0000","last_undegraded":"2024-09-13T14:12:38.182212+0000","last_fullsized":"2024-09-13T14:12:38.182212+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:14.744825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042041700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"53'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:12:31.171615+0000","last_change":"2024-09-13T14:11:54.779008+0000","last_active":"2024-09-13T14:12:31.171615+0000","last_peered":"2024-09-13T14:12:31.171615+0000","last_clean":"2024-09-13T14:12:31.171615+0000","last_became_active":"2024-09-13T14:11:54.772279+0000","last_became_peered":"2024-09-13T14:11:54.772279+0000","last_unstale":"2024-09-13T14:12:31.171615+0000","last_undegraded":"2024-09-13T14:12:31.171615+0000","last_fullsized":"2024-09-13T14:12:31.171615+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:02.973115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397526,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61399999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55500000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.50900000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.63200000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.71699999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.73099999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.59799999999999998}]}]},{"osd":6,"up_from":43,"seq":184683593754,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.379}]},{"osd":1,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47899999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.46000000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.33100000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.439}]},{"osd":7,"last update":"Fri Sep 13 14:12:52 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.51200000000000001}]}]},{"osd":5,"up_from":37,"seq":158913789984,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":245477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.67600000000000005,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.55700000000000005,"15min":0.56799999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.64000000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.54000000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55900000000000005,"15min":0.56399999999999995},"min":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69799999999999995,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.55900000000000005,"15min":0.56599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70199999999999996,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56399999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72399999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.56299999999999994,"15min":0.56799999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.51700000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39200000000000002,"15min":0.38900000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.67700000000000005,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.372},{"interface":"front","average":{"1min":0.439,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.41799999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.441}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.36099999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.34300000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.498}]},{"osd":1,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52000000000000002,"15min":0.52200000000000002},"min":{"1min":0.374,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.51700000000000002,"15min":0.52100000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63100000000000001,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.51700000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51500000000000001,"15min":0.51300000000000001},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.311},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.38200000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.64100000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:12:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52900000000000003,"15min":0.53100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.51000000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.57699999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:13:09 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.34799999999999998,"15min":0.33500000000000002},"min":{"1min":0.246,"5min":0.22,"15min":0.22},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.35799999999999998,"15min":0.34300000000000003},"min":{"1min":0.27000000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.439}]},{"osd":6,"last update":"Fri Sep 13 14:12:29 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.56200000000000006}]},{"osd":7,"last update":"Fri Sep 13 14:12:52 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.62}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:27 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.374,"15min":0.376},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":0.58399999999999996,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39800000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.54100000000000004,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.39600000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:13:27 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.38400000000000001,"15min":0.39000000000000001},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.505,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.373}]},{"osd":2,"last update":"Fri Sep 13 14:13:27 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60399999999999998,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45200000000000001,"15min":0.46000000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.628,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.44800000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:12:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57799999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84999999999999998,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.72199999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:13:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.64900000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.76600000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.755}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.374,"15min":0.36599999999999999},"min":{"1min":0.29799999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.29399999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.53700000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.29099999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38200000000000001,"15min":0.376},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51300000000000001,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.41799999999999998,"15min":0.41299999999999998},"min":{"1min":0.36299999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.627,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.27200000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.45100000000000001,"15min":0.46000000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.54400000000000004,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.442},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.435,"15min":0.44900000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.56799999999999995,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.32500000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:12:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.60399999999999998,"15min":0.59399999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.505},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.52700000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:13:05 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.56899999999999995,"15min":0.56299999999999994},"min":{"1min":0.42199999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.46000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73499999999999999,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.41899999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.47199999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.42899999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35699999999999998,"15min":0.34699999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57499999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.245,"5min":0.214,"15min":0.214},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40500000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38800000000000001,"15min":0.38700000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.53000000000000003,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40999999999999998,"15min":0.40500000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.54500000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.42699999999999999,"15min":0.437},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.52200000000000002,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.442,"15min":0.45100000000000001},"min":{"1min":0.29599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.55100000000000005,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.48499999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:12:42 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.878,"15min":0.878},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.44700000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81699999999999995,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.55700000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:13:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59299999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.54000000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.504}]},{"osd":7,"last update":"Fri Sep 13 14:12:53 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.624},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.59699999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":836141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:12:40 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.34999999999999998,"15min":0.33900000000000002},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.51500000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.35999999999999999,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.255}]},{"osd":2,"last update":"Fri Sep 13 14:13:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38700000000000001,"15min":0.38300000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.40799999999999997},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.29799999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:13:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44400000000000001,"15min":0.44600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46000000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:12:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.502},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53200000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:13:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.625},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.57999999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.46500000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.55100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:13:32.772 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:13:32.773 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:13:32.773 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:13:32.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:13:33.373 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:33.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:33 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:13:34.049 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:33 smithi017 podman[74567]: 2024-09-13 14:13:33.804288255 +0000 UTC m=+0.432361992 container died bd30c26b37f8bd1cce938febe3b97b6e8e5ec05bf91b6f05d62ee9b42c60753d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=sleepy_sammet, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T14:13:34.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:34 smithi017 ceph-mon[29399]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:34.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:34 smithi017 ceph-mon[38255]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:34.684 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:34 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[69031]: ts=2024-09-13T14:13:34.496Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:13:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:34 smithi189 ceph-mon[33294]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:34.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:34 smithi017 podman[74649]: 2024-09-13 14:13:34.696349479 +0000 UTC m=+0.729328356 container died fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:35.247 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:35 smithi017 podman[74649]: 2024-09-13 14:13:35.029289105 +0000 UTC m=+1.062268023 container cleanup fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:35.248 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:35 smithi017 bash[74649]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:13:35.248 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:35 smithi017 podman[74693]: 2024-09-13 14:13:35.188486987 +0000 UTC m=+0.672979705 container remove fdf91de48026cb40d684d8220f8aef387601bf630deb242eb5deaa3e297d75e4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:35.248 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:35 smithi017 podman[74693]: 2024-09-13 14:13:35.247531862 +0000 UTC m=+0.732024584 volume remove 65d914028e8c647218223f2adb5b9c0bbfbd37b70e2728271adacd864899229e 2024-09-13T14:13:35.646 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:35.646 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:13:36.264 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:13:36.264 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T14:13:36.264 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T14:13:36.277 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2024-09-13T14:13:36.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-13T14:13:36.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-mon[29399]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:36.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2359630876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:13:36.570 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:13:36.570 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:36.570 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 systemd[1]: Starting Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:13:36.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-mon[38255]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:36.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2359630876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:13:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:36 smithi189 ceph-mon[33294]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:36 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2359630876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:13:36.853 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 podman[74873]: 2024-09-13 14:13:36.569602947 +0000 UTC m=+0.111936202 volume create 58f022f0222017e75546a49244c932f8ecd14de1cbb422468ef320928fa09d4d 2024-09-13T14:13:36.853 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 podman[74873]: 2024-09-13 14:13:36.483665307 +0000 UTC m=+0.025998569 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T14:13:36.853 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 podman[74873]: 2024-09-13 14:13:36.670115884 +0000 UTC m=+0.212449178 container create c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:37.106 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:37.230 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 podman[74873]: 2024-09-13 14:13:36.964594228 +0000 UTC m=+0.506927486 container init c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 podman[74873]: 2024-09-13 14:13:36.969995014 +0000 UTC m=+0.512328271 container start c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:36.983Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:36.983Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:36.984Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:36 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:36.985Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:37.008Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:37.009Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:37.012Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:37.012Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 bash[74873]: c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 2024-09-13T14:13:37.231 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:37 smithi017 systemd[1]: Started Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:38.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[29399]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:13:38.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[29399]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:13:38.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[29399]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[38255]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:13:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[38255]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:13:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-mon[38255]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:38 smithi189 ceph-mon[33294]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:13:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:38 smithi189 ceph-mon[33294]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:13:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:38 smithi189 ceph-mon[33294]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:39.106 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:38.986Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000447391s 2024-09-13T14:13:39.288 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:39 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:39 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:39 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:39.541 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:40.117 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-mon[33294]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:40.118 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:13:40.118 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-mon[33294]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[38255]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[38255]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[29399]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:13:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:40 smithi017 ceph-mon[29399]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 systemd[1]: Stopping Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:13:40.833 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.575Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.576Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.576Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.576Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[62342]: ts=2024-09-13T14:13:40.576Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:13:40.834 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:40 smithi189 podman[65663]: 2024-09-13 14:13:40.585936501 +0000 UTC m=+0.132106259 container died e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:41.113 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 podman[65663]: 2024-09-13 14:13:41.023448891 +0000 UTC m=+0.569618651 container cleanup e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:41.113 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 bash[65663]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:13:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:13:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[38255]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:13:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:13:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[29399]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:13:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:41 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[29376]: 2024-09-13T14:13:41.125+0000 7f171ff8c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:13:41.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 podman[65682]: 2024-09-13 14:13:41.317618917 +0000 UTC m=+0.733324560 container remove e2fac2cf3926def52418f5937980947dcd6b0c87d3c74c9bad696c9ca07884bd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:41 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:13:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:41 smithi189 ceph-mon[33294]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:13:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:41 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:41.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 systemd[1]: Stopped Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.088s CPU time. 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 systemd[1]: Starting Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 podman[65774]: 2024-09-13 14:13:41.706228064 +0000 UTC m=+0.017025453 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:13:41.931 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:41 smithi189 podman[65774]: 2024-09-13 14:13:41.818523754 +0000 UTC m=+0.129321180 container create acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:42.077 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:13:42.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: fsmap foo:0 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: Saving service mds.foo spec with placement count:2 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[38255]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STOPPING 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STOPPED 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STARTING 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Serving on http://:::9283 2024-09-13T14:13:42.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STARTED 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STOPPING 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: fsmap foo:0 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: Saving service mds.foo spec with placement count:2 2024-09-13T14:13:42.379 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:42.380 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-mon[29399]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:13:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:13:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: fsmap foo:0 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: Saving service mds.foo spec with placement count:2 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-mon[33294]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 podman[65774]: 2024-09-13 14:13:42.029028374 +0000 UTC m=+0.339825760 container init acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 podman[65774]: 2024-09-13 14:13:42.033870922 +0000 UTC m=+0.344668306 container start acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.057Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.057Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.057Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi189 (none))" 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.057Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.057Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.059Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.059Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.061Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.061Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:13:42.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.535µs 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.063Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.338µs wal_replay_duration=567.041µs wbl_replay_duration=168ns total_replay_duration=604.306µs 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.064Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.064Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.064Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.079Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.172587ms db_storage=689ns remote_storage=994ns web_handler=273ns query_engine=608ns scrape=589.493µs scrape_sd=29.79µs notify=24.722µs notify_sd=7.544µs rules=14.287976ms tracing=6.287µs 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.079Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:13:42.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:13:42.079Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:13:42.509 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 bash[65774]: acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 2024-09-13T14:13:42.509 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:13:42 smithi189 systemd[1]: Started Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:13:43.137 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:13:43.137 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STOPPED 2024-09-13T14:13:43.137 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STARTING 2024-09-13T14:13:43.137 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Serving on http://:::9283 2024-09-13T14:13:43.138 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STARTED 2024-09-13T14:13:43.138 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:42 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:42] ENGINE Bus STOPPING 2024-09-13T14:13:43.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:13:43.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:43.478 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.479 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:13:43.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:13:43.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:43.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:43 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:43.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:13:43.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:43] ENGINE Bus STOPPED 2024-09-13T14:13:43.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:43] ENGINE Bus STARTING 2024-09-13T14:13:43.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:43] ENGINE Serving on http://:::9283 2024-09-13T14:13:43.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:43 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:13:43] ENGINE Bus STARTED 2024-09-13T14:13:44.344 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:44 smithi017 ceph-mon[29399]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:13:44.344 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:44 smithi017 ceph-mon[29399]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:44.344 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:44 smithi017 ceph-mon[38255]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:13:44.344 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:44 smithi017 ceph-mon[38255]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:44 smithi189 ceph-mon[33294]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:13:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:44 smithi189 ceph-mon[33294]: pgmap v55: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:44.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T14:13:45.221 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:46.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[29399]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:46.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:46.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:46.665 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[38255]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:46 smithi189 ceph-mon[33294]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:46 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:46 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:47.295 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:13:46 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:13:46.990Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004002091s 2024-09-13T14:13:47.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:47] "GET /metrics HTTP/1.1" 200 203807 "" "Prometheus/2.43.0" 2024-09-13T14:13:48.309 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:13:48.320 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:13:48.320 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:13:48.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:13:48.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:48 smithi017 ceph-mon[29399]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:48.610 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:48 smithi017 ceph-mon[38255]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:48 smithi189 ceph-mon[33294]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:49.077 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:49.999 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: Deploying daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[29399]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: Deploying daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:13:50.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:49 smithi017 ceph-mon[38255]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:50.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: Deploying daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:13:50.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:49 smithi189 ceph-mon[33294]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:51.478 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:51.478 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:51 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2767439695' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:13:51.479 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:51 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2767439695' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:13:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:51 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2767439695' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:13:52.061 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":21,"flags":0,"active_gid":14412,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":2742834835},{"type":"v1","addr":"172.21.15.17:6801","nonce":2742834835}]},"active_addr":"172.21.15.17:6801/2742834835","active_change":"2024-09-13T14:12:23.277867+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1515064414}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":101070012}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3476899836}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2755559292}]}]} 2024-09-13T14:13:52.065 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:13:52.065 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:13:52.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:13:52.462 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:52.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:52 smithi017 ceph-mon[29399]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:52.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:52 smithi017 ceph-mon[38255]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:52 smithi189 ceph-mon[33294]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:54.498 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[29399]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:54.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:54.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:54.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[38255]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:54.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:54.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:54 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:54.574 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:54.574 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":64,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:13:43.149265+0000","last_up_change":"2024-09-13T14:11:52.754486+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6803","nonce":2866456241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6805","nonce":2866456241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6809","nonce":2866456241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6807","nonce":2866456241}]},"public_addr":"172.21.15.17:6803/2866456241","cluster_addr":"172.21.15.17:6805/2866456241","heartbeat_back_addr":"172.21.15.17:6809/2866456241","heartbeat_front_addr":"172.21.15.17:6807/2866456241","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6811","nonce":3915572123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6813","nonce":3915572123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6817","nonce":3915572123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6815","nonce":3915572123}]},"public_addr":"172.21.15.17:6811/3915572123","cluster_addr":"172.21.15.17:6813/3915572123","heartbeat_back_addr":"172.21.15.17:6817/3915572123","heartbeat_front_addr":"172.21.15.17:6815/3915572123","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6819","nonce":1775042177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6821","nonce":1775042177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6825","nonce":1775042177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6823","nonce":1775042177}]},"public_addr":"172.21.15.17:6819/1775042177","cluster_addr":"172.21.15.17:6821/1775042177","heartbeat_back_addr":"172.21.15.17:6825/1775042177","heartbeat_front_addr":"172.21.15.17:6823/1775042177","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6827","nonce":2937054127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6829","nonce":2937054127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6833","nonce":2937054127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6831","nonce":2937054127}]},"public_addr":"172.21.15.17:6827/2937054127","cluster_addr":"172.21.15.17:6829/2937054127","heartbeat_back_addr":"172.21.15.17:6833/2937054127","heartbeat_front_addr":"172.21.15.17:6831/2937054127","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:13:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:54 smithi189 ceph-mon[33294]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:54 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:54 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:13:55.344 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:13:55.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1842011691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:55.346 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:55 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:55.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1842011691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1842011691' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:13:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:13:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:55 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:56.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: Deploying daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:boot 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: daemon mds.foo.smithi017.ldrmqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: Cluster is now healthy 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: fsmap foo:0 1 up:standby 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:creating} 2024-09-13T14:13:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:56 smithi189 ceph-mon[33294]: daemon mds.foo.smithi017.ldrmqj is now active in filesystem foo as rank 0 2024-09-13T14:13:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: Deploying daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:13:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:boot 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: daemon mds.foo.smithi017.ldrmqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: Cluster is now healthy 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: fsmap foo:0 1 up:standby 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:creating} 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[29399]: daemon mds.foo.smithi017.ldrmqj is now active in filesystem foo as rank 0 2024-09-13T14:13:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: Deploying daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:boot 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: daemon mds.foo.smithi017.ldrmqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: Cluster is now healthy 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: fsmap foo:0 1 up:standby 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:creating} 2024-09-13T14:13:56.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:56 smithi017 ceph-mon[38255]: daemon mds.foo.smithi017.ldrmqj is now active in filesystem foo as rank 0 2024-09-13T14:13:57.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:57.478 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:13:57.478 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":64,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:13:43.149265+0000","last_up_change":"2024-09-13T14:11:52.754486+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6803","nonce":2866456241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6805","nonce":2866456241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6809","nonce":2866456241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2866456241},{"type":"v1","addr":"172.21.15.17:6807","nonce":2866456241}]},"public_addr":"172.21.15.17:6803/2866456241","cluster_addr":"172.21.15.17:6805/2866456241","heartbeat_back_addr":"172.21.15.17:6809/2866456241","heartbeat_front_addr":"172.21.15.17:6807/2866456241","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6811","nonce":3915572123}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6813","nonce":3915572123}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6817","nonce":3915572123}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3915572123},{"type":"v1","addr":"172.21.15.17:6815","nonce":3915572123}]},"public_addr":"172.21.15.17:6811/3915572123","cluster_addr":"172.21.15.17:6813/3915572123","heartbeat_back_addr":"172.21.15.17:6817/3915572123","heartbeat_front_addr":"172.21.15.17:6815/3915572123","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6819","nonce":1775042177}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6821","nonce":1775042177}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6825","nonce":1775042177}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1775042177},{"type":"v1","addr":"172.21.15.17:6823","nonce":1775042177}]},"public_addr":"172.21.15.17:6819/1775042177","cluster_addr":"172.21.15.17:6821/1775042177","heartbeat_back_addr":"172.21.15.17:6825/1775042177","heartbeat_front_addr":"172.21.15.17:6823/1775042177","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6827","nonce":2937054127}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6829","nonce":2937054127}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6833","nonce":2937054127}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":2937054127},{"type":"v1","addr":"172.21.15.17:6831","nonce":2937054127}]},"public_addr":"172.21.15.17:6827/2937054127","cluster_addr":"172.21.15.17:6829/2937054127","heartbeat_back_addr":"172.21.15.17:6833/2937054127","heartbeat_front_addr":"172.21.15.17:6831/2937054127","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:13:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:13:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 2024-09-13T14:13:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:57 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:58.126 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:13:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:57] "GET /metrics HTTP/1.1" 200 208795 "" "Prometheus/2.43.0" 2024-09-13T14:13:58.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:13:58.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:13:58.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:13:58.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:13:58.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:13:58.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:13:58.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:13:58.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:boot 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/97009851' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:boot 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:58 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/97009851' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:boot 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:13:58.547 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:58 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/97009851' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:13:59.116 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.152 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.153 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.156 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.157 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.167 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.171 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.172 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:59.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:13:59 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:59.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:13:59 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:13:59.653 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:13:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:13:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:13:59.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:13:59 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:00.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:00 smithi017 ceph-mon[29399]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:00 smithi017 ceph-mon[38255]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:00 smithi189 ceph-mon[33294]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:01.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[29399]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:14:01.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:01.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:01.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:01.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[38255]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:14:01.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:01.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:01.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:01 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:01 smithi189 ceph-mon[33294]: mds.? [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] up:active 2024-09-13T14:14:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:01 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:01 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:01 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[29399]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[29399]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:standby 2024-09-13T14:14:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[29399]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[38255]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[38255]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:standby 2024-09-13T14:14:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:02 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:02 smithi189 ceph-mon[33294]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:02 smithi189 ceph-mon[33294]: mds.? [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] up:standby 2024-09-13T14:14:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:02 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:14:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:04 smithi017 ceph-mon[29399]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:04 smithi017 ceph-mon[38255]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:04 smithi189 ceph-mon[33294]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:06.334 INFO:teuthology.orchestra.run.smithi017.stdout:158913789992 2024-09-13T14:14:06.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:14:06.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:06 smithi017 ceph-mon[29399]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:06.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:06 smithi017 ceph-mon[38255]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:06 smithi189 ceph-mon[33294]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:08.130 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:07] "GET /metrics HTTP/1.1" 200 208795 "" "Prometheus/2.43.0" 2024-09-13T14:14:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[29399]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[38255]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:08 smithi189 ceph-mon[33294]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:08.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:10.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:14:10.214 INFO:teuthology.orchestra.run.smithi017.stdout:133143986221 2024-09-13T14:14:10.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:14:10.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:10 smithi017 ceph-mon[29399]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:10.439 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:10 smithi017 ceph-mon[38255]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:10.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:10 smithi189 ceph-mon[33294]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T14:14:12.174 INFO:teuthology.orchestra.run.smithi017.stdout:111669149743 2024-09-13T14:14:12.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:14:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:12 smithi017 ceph-mon[29399]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:12 smithi017 ceph-mon[38255]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:12 smithi189 ceph-mon[33294]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:13.086 INFO:teuthology.orchestra.run.smithi017.stdout:55834574907 2024-09-13T14:14:13.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:14:14.496 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:14 smithi017 ceph-mon[29399]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:14.496 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:14 smithi017 ceph-mon[38255]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:14 smithi189 ceph-mon[33294]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:15.309 INFO:teuthology.orchestra.run.smithi017.stdout:34359738432 2024-09-13T14:14:15.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:14:16.139 INFO:teuthology.orchestra.run.smithi017.stdout:210453397535 2024-09-13T14:14:16.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:14:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:16 smithi017 ceph-mon[29399]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:16 smithi017 ceph-mon[38255]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:16 smithi189 ceph-mon[33294]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:14:17.147 INFO:teuthology.orchestra.run.smithi017.stdout:184683593763 2024-09-13T14:14:17.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:14:18.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:17] "GET /metrics HTTP/1.1" 200 239772 "" "Prometheus/2.43.0" 2024-09-13T14:14:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:18 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:18.677 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:18.677 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:18.679 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:18.714 INFO:teuthology.orchestra.run.smithi017.stdout:77309411383 2024-09-13T14:14:18.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:14:18.745 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:18.751 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:18 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:19.841 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:14:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:20 smithi017 ceph-mon[29399]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:20 smithi017 ceph-mon[38255]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:20.500 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:20.500 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:20 smithi189 ceph-mon[33294]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[29399]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[38255]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:22 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:22 smithi189 ceph-mon[33294]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:22 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:22 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-13T14:14:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-13T14:14:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-13T14:14:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T14:14:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:24.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[29399]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[29399]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[38255]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T14:14:24.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:24 smithi017 ceph-mon[38255]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:14:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:24 smithi189 ceph-mon[33294]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:14:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-13T14:14:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:24 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T14:14:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:24 smithi189 ceph-mon[33294]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:14:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:25 smithi017 ceph-mon[29399]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:14:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:25 smithi017 ceph-mon[38255]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:14:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:25 smithi189 ceph-mon[33294]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:14:26.256 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:26 smithi017 ceph-mon[29399]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:26 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3873305759' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:14:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:26 smithi017 ceph-mon[38255]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:26 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3873305759' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:14:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:26 smithi189 ceph-mon[33294]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:26 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3873305759' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:14:27.555 INFO:teuthology.orchestra.run.smithi017.stdout:133143986224 2024-09-13T14:14:28.115 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:27] "GET /metrics HTTP/1.1" 200 239761 "" "Prometheus/2.43.0" 2024-09-13T14:14:28.146 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986221 got 133143986224 for osd.4 2024-09-13T14:14:28.146 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:28.233 INFO:teuthology.orchestra.run.smithi017.stdout:158913789996 2024-09-13T14:14:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:28 smithi017 ceph-mon[38255]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:28 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3504662031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:14:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:28 smithi017 ceph-mon[29399]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:28 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3504662031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:14:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:28 smithi189 ceph-mon[33294]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:28 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3504662031' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:14:28.916 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789992 got 158913789996 for osd.5 2024-09-13T14:14:28.917 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:29.787 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:14:30.357 INFO:teuthology.orchestra.run.smithi017.stdout:34359738435 2024-09-13T14:14:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:30 smithi017 ceph-mon[29399]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:30 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/578009392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:14:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:30 smithi017 ceph-mon[38255]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:30 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/578009392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:14:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:30 smithi189 ceph-mon[33294]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:30 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/578009392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:14:31.489 INFO:teuthology.orchestra.run.smithi017.stdout:184683593767 2024-09-13T14:14:32.067 INFO:teuthology.orchestra.run.smithi017.stdout:111669149748 2024-09-13T14:14:32.320 INFO:teuthology.orchestra.run.smithi017.stdout:55834574911 2024-09-13T14:14:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[29399]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2420847788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2546588460' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2113731227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[38255]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2420847788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2546588460' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:14:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:32 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2113731227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:14:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:32 smithi189 ceph-mon[33294]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:32 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2420847788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:14:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:32 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2546588460' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:14:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:32 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2113731227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:14:32.914 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149743 got 111669149748 for osd.3 2024-09-13T14:14:32.914 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:33.397 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593763 got 184683593767 for osd.6 2024-09-13T14:14:33.397 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:34.344 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574911 for osd.1 2024-09-13T14:14:34.344 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:34.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:34 smithi017 ceph-mon[29399]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:34.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:34 smithi017 ceph-mon[38255]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:34 smithi189 ceph-mon[33294]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:35.078 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738435 for osd.0 2024-09-13T14:14:35.078 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:35.454 INFO:teuthology.orchestra.run.smithi017.stdout:210453397540 2024-09-13T14:14:36.297 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397535 got 210453397540 for osd.7 2024-09-13T14:14:36.297 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:36.687 INFO:teuthology.orchestra.run.smithi017.stdout:77309411387 2024-09-13T14:14:36.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:36 smithi017 ceph-mon[29399]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:36.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:36 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1486350354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:14:36.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:36 smithi017 ceph-mon[38255]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:36.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:36 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1486350354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:14:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:36 smithi189 ceph-mon[33294]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:36 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1486350354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:14:37.301 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411387 for osd.2 2024-09-13T14:14:37.301 DEBUG:teuthology.parallel:result is None 2024-09-13T14:14:37.301 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:14:37.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:14:37.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2931482574' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:14:37.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2931482574' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:14:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2931482574' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:14:37.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:37] "GET /metrics HTTP/1.1" 200 239761 "" "Prometheus/2.43.0" 2024-09-13T14:14:37.991 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:38 smithi017 ceph-mon[29399]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:38 smithi017 ceph-mon[38255]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:38 smithi189 ceph-mon[33294]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:39.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:14:40.429 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:14:40.431 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:14:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[29399]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[38255]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:40 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:40 smithi189 ceph-mon[33294]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:40 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:40 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:41.207 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-09-13T14:14:39.314255+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385928,"kb_used_data":7076,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575288,"statfs":{"total":767960285184,"available":765517094912,"internally_reserved":0,"allocated":7245824,"data_stored":4148727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002563"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398058+0000","last_change":"2024-09-13T14:13:42.151465+0000","last_active":"2024-09-13T14:14:23.398058+0000","last_peered":"2024-09-13T14:14:23.398058+0000","last_clean":"2024-09-13T14:14:23.398058+0000","last_became_active":"2024-09-13T14:13:42.151206+0000","last_became_peered":"2024-09-13T14:13:42.151206+0000","last_unstale":"2024-09-13T14:14:23.398058+0000","last_undegraded":"2024-09-13T14:14:23.398058+0000","last_fullsized":"2024-09-13T14:14:23.398058+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:14.848155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433717+0000","last_change":"2024-09-13T14:13:41.136356+0000","last_active":"2024-09-13T14:14:23.433717+0000","last_peered":"2024-09-13T14:14:23.433717+0000","last_clean":"2024-09-13T14:14:23.433717+0000","last_became_active":"2024-09-13T14:13:41.136101+0000","last_became_peered":"2024-09-13T14:13:41.136101+0000","last_unstale":"2024-09-13T14:14:23.433717+0000","last_undegraded":"2024-09-13T14:14:23.433717+0000","last_fullsized":"2024-09-13T14:14:23.433717+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:44.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401729+0000","last_change":"2024-09-13T14:13:42.150106+0000","last_active":"2024-09-13T14:14:23.401729+0000","last_peered":"2024-09-13T14:14:23.401729+0000","last_clean":"2024-09-13T14:14:23.401729+0000","last_became_active":"2024-09-13T14:13:42.149808+0000","last_became_peered":"2024-09-13T14:13:42.149808+0000","last_unstale":"2024-09-13T14:14:23.401729+0000","last_undegraded":"2024-09-13T14:14:23.401729+0000","last_fullsized":"2024-09-13T14:14:23.401729+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:21.087262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403365+0000","last_change":"2024-09-13T14:13:41.131071+0000","last_active":"2024-09-13T14:14:23.403365+0000","last_peered":"2024-09-13T14:14:23.403365+0000","last_clean":"2024-09-13T14:14:23.403365+0000","last_became_active":"2024-09-13T14:13:41.130771+0000","last_became_peered":"2024-09-13T14:13:41.130771+0000","last_unstale":"2024-09-13T14:14:23.403365+0000","last_undegraded":"2024-09-13T14:14:23.403365+0000","last_fullsized":"2024-09-13T14:14:23.403365+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:32.111331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403708+0000","last_change":"2024-09-13T14:13:42.149841+0000","last_active":"2024-09-13T14:14:23.403708+0000","last_peered":"2024-09-13T14:14:23.403708+0000","last_clean":"2024-09-13T14:14:23.403708+0000","last_became_active":"2024-09-13T14:13:42.149564+0000","last_became_peered":"2024-09-13T14:13:42.149564+0000","last_unstale":"2024-09-13T14:14:23.403708+0000","last_undegraded":"2024-09-13T14:14:23.403708+0000","last_fullsized":"2024-09-13T14:14:23.403708+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:58.753171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433655+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:14:23.433655+0000","last_peered":"2024-09-13T14:14:23.433655+0000","last_clean":"2024-09-13T14:14:23.433655+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:14:23.433655+0000","last_undegraded":"2024-09-13T14:14:23.433655+0000","last_fullsized":"2024-09-13T14:14:23.433655+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398171+0000","last_change":"2024-09-13T14:13:42.151443+0000","last_active":"2024-09-13T14:14:23.398171+0000","last_peered":"2024-09-13T14:14:23.398171+0000","last_clean":"2024-09-13T14:14:23.398171+0000","last_became_active":"2024-09-13T14:13:42.151156+0000","last_became_peered":"2024-09-13T14:13:42.151156+0000","last_unstale":"2024-09-13T14:14:23.398171+0000","last_undegraded":"2024-09-13T14:14:23.398171+0000","last_fullsized":"2024-09-13T14:14:23.398171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:18.895996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401095+0000","last_change":"2024-09-13T14:13:41.130289+0000","last_active":"2024-09-13T14:14:23.401095+0000","last_peered":"2024-09-13T14:14:23.401095+0000","last_clean":"2024-09-13T14:14:23.401095+0000","last_became_active":"2024-09-13T14:13:41.129333+0000","last_became_peered":"2024-09-13T14:13:41.129333+0000","last_unstale":"2024-09-13T14:14:23.401095+0000","last_undegraded":"2024-09-13T14:14:23.401095+0000","last_fullsized":"2024-09-13T14:14:23.401095+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:16.787146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.434023+0000","last_change":"2024-09-13T14:13:41.132344+0000","last_active":"2024-09-13T14:14:23.434023+0000","last_peered":"2024-09-13T14:14:23.434023+0000","last_clean":"2024-09-13T14:14:23.434023+0000","last_became_active":"2024-09-13T14:13:41.132064+0000","last_became_peered":"2024-09-13T14:13:41.132064+0000","last_unstale":"2024-09-13T14:14:23.434023+0000","last_undegraded":"2024-09-13T14:14:23.434023+0000","last_fullsized":"2024-09-13T14:14:23.434023+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:19:00.797682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400890+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:14:23.400890+0000","last_peered":"2024-09-13T14:14:23.400890+0000","last_clean":"2024-09-13T14:14:23.400890+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:14:23.400890+0000","last_undegraded":"2024-09-13T14:14:23.400890+0000","last_fullsized":"2024-09-13T14:14:23.400890+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.434017+0000","last_change":"2024-09-13T14:13:41.130783+0000","last_active":"2024-09-13T14:14:23.434017+0000","last_peered":"2024-09-13T14:14:23.434017+0000","last_clean":"2024-09-13T14:14:23.434017+0000","last_became_active":"2024-09-13T14:13:41.130531+0000","last_became_peered":"2024-09-13T14:13:41.130531+0000","last_unstale":"2024-09-13T14:14:23.434017+0000","last_undegraded":"2024-09-13T14:14:23.434017+0000","last_fullsized":"2024-09-13T14:14:23.434017+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:01.706615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398278+0000","last_change":"2024-09-13T14:13:41.150904+0000","last_active":"2024-09-13T14:14:23.398278+0000","last_peered":"2024-09-13T14:14:23.398278+0000","last_clean":"2024-09-13T14:14:23.398278+0000","last_became_active":"2024-09-13T14:13:41.150184+0000","last_became_peered":"2024-09-13T14:13:41.150184+0000","last_unstale":"2024-09-13T14:14:23.398278+0000","last_undegraded":"2024-09-13T14:14:23.398278+0000","last_fullsized":"2024-09-13T14:14:23.398278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:01.674973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433973+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:14:23.433973+0000","last_peered":"2024-09-13T14:14:23.433973+0000","last_clean":"2024-09-13T14:14:23.433973+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:14:23.433973+0000","last_undegraded":"2024-09-13T14:14:23.433973+0000","last_fullsized":"2024-09-13T14:14:23.433973+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400674+0000","last_change":"2024-09-13T14:13:41.147751+0000","last_active":"2024-09-13T14:14:23.400674+0000","last_peered":"2024-09-13T14:14:23.400674+0000","last_clean":"2024-09-13T14:14:23.400674+0000","last_became_active":"2024-09-13T14:13:41.147448+0000","last_became_peered":"2024-09-13T14:13:41.147448+0000","last_unstale":"2024-09-13T14:14:23.400674+0000","last_undegraded":"2024-09-13T14:14:23.400674+0000","last_fullsized":"2024-09-13T14:14:23.400674+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:50.145254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401397+0000","last_change":"2024-09-13T14:13:41.131269+0000","last_active":"2024-09-13T14:14:23.401397+0000","last_peered":"2024-09-13T14:14:23.401397+0000","last_clean":"2024-09-13T14:14:23.401397+0000","last_became_active":"2024-09-13T14:13:41.131031+0000","last_became_peered":"2024-09-13T14:13:41.131031+0000","last_unstale":"2024-09-13T14:14:23.401397+0000","last_undegraded":"2024-09-13T14:14:23.401397+0000","last_fullsized":"2024-09-13T14:14:23.401397+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:33.641297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403511+0000","last_change":"2024-09-13T14:13:42.607860+0000","last_active":"2024-09-13T14:14:23.403511+0000","last_peered":"2024-09-13T14:14:23.403511+0000","last_clean":"2024-09-13T14:14:23.403511+0000","last_became_active":"2024-09-13T14:13:42.607517+0000","last_became_peered":"2024-09-13T14:13:42.607517+0000","last_unstale":"2024-09-13T14:14:23.403511+0000","last_undegraded":"2024-09-13T14:14:23.403511+0000","last_fullsized":"2024-09-13T14:14:23.403511+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:17:06.998105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398088+0000","last_change":"2024-09-13T14:13:41.148026+0000","last_active":"2024-09-13T14:14:23.398088+0000","last_peered":"2024-09-13T14:14:23.398088+0000","last_clean":"2024-09-13T14:14:23.398088+0000","last_became_active":"2024-09-13T14:13:41.146192+0000","last_became_peered":"2024-09-13T14:13:41.146192+0000","last_unstale":"2024-09-13T14:14:23.398088+0000","last_undegraded":"2024-09-13T14:14:23.398088+0000","last_fullsized":"2024-09-13T14:14:23.398088+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:17.333126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398384+0000","last_change":"2024-09-13T14:13:41.144813+0000","last_active":"2024-09-13T14:14:23.398384+0000","last_peered":"2024-09-13T14:14:23.398384+0000","last_clean":"2024-09-13T14:14:23.398384+0000","last_became_active":"2024-09-13T14:13:41.144542+0000","last_became_peered":"2024-09-13T14:13:41.144542+0000","last_unstale":"2024-09-13T14:14:23.398384+0000","last_undegraded":"2024-09-13T14:14:23.398384+0000","last_fullsized":"2024-09-13T14:14:23.398384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:29.013377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400576+0000","last_change":"2024-09-13T14:13:41.126997+0000","last_active":"2024-09-13T14:14:23.400576+0000","last_peered":"2024-09-13T14:14:23.400576+0000","last_clean":"2024-09-13T14:14:23.400576+0000","last_became_active":"2024-09-13T14:13:41.126723+0000","last_became_peered":"2024-09-13T14:13:41.126723+0000","last_unstale":"2024-09-13T14:14:23.400576+0000","last_undegraded":"2024-09-13T14:14:23.400576+0000","last_fullsized":"2024-09-13T14:14:23.400576+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:06.360290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.419739+0000","last_change":"2024-09-13T14:14:24.419739+0000","last_active":"2024-09-13T14:14:24.419739+0000","last_peered":"2024-09-13T14:14:24.419739+0000","last_clean":"2024-09-13T14:14:24.419739+0000","last_became_active":"2024-09-13T14:14:24.419396+0000","last_became_peered":"2024-09-13T14:14:24.419396+0000","last_unstale":"2024-09-13T14:14:24.419739+0000","last_undegraded":"2024-09-13T14:14:24.419739+0000","last_fullsized":"2024-09-13T14:14:24.419739+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:11.051977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398400+0000","last_change":"2024-09-13T14:13:41.133374+0000","last_active":"2024-09-13T14:14:23.398400+0000","last_peered":"2024-09-13T14:14:23.398400+0000","last_clean":"2024-09-13T14:14:23.398400+0000","last_became_active":"2024-09-13T14:13:41.132989+0000","last_became_peered":"2024-09-13T14:13:41.132989+0000","last_unstale":"2024-09-13T14:14:23.398400+0000","last_undegraded":"2024-09-13T14:14:23.398400+0000","last_fullsized":"2024-09-13T14:14:23.398400+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:57.357479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398589+0000","last_change":"2024-09-13T14:13:41.151154+0000","last_active":"2024-09-13T14:14:23.398589+0000","last_peered":"2024-09-13T14:14:23.398589+0000","last_clean":"2024-09-13T14:14:23.398589+0000","last_became_active":"2024-09-13T14:13:41.150785+0000","last_became_peered":"2024-09-13T14:13:41.150785+0000","last_unstale":"2024-09-13T14:14:23.398589+0000","last_undegraded":"2024-09-13T14:14:23.398589+0000","last_fullsized":"2024-09-13T14:14:23.398589+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:18.759670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397779+0000","last_change":"2024-09-13T14:13:41.139264+0000","last_active":"2024-09-13T14:14:23.397779+0000","last_peered":"2024-09-13T14:14:23.397779+0000","last_clean":"2024-09-13T14:14:23.397779+0000","last_became_active":"2024-09-13T14:13:41.138964+0000","last_became_peered":"2024-09-13T14:13:41.138964+0000","last_unstale":"2024-09-13T14:14:23.397779+0000","last_undegraded":"2024-09-13T14:14:23.397779+0000","last_fullsized":"2024-09-13T14:14:23.397779+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:32:51.533046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401184+0000","last_change":"2024-09-13T14:13:41.153668+0000","last_active":"2024-09-13T14:14:23.401184+0000","last_peered":"2024-09-13T14:14:23.401184+0000","last_clean":"2024-09-13T14:14:23.401184+0000","last_became_active":"2024-09-13T14:13:41.153316+0000","last_became_peered":"2024-09-13T14:13:41.153316+0000","last_unstale":"2024-09-13T14:14:23.401184+0000","last_undegraded":"2024-09-13T14:14:23.401184+0000","last_fullsized":"2024-09-13T14:14:23.401184+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:30:23.303445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"53'79","reported_seq":56,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433730+0000","last_change":"2024-09-13T14:11:54.779008+0000","last_active":"2024-09-13T14:14:23.433730+0000","last_peered":"2024-09-13T14:14:23.433730+0000","last_clean":"2024-09-13T14:14:23.433730+0000","last_became_active":"2024-09-13T14:11:54.772279+0000","last_became_peered":"2024-09-13T14:11:54.772279+0000","last_unstale":"2024-09-13T14:14:23.433730+0000","last_undegraded":"2024-09-13T14:14:23.433730+0000","last_fullsized":"2024-09-13T14:14:23.433730+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:02.973115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":27,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401073+0000","last_change":"2024-09-13T14:12:31.085929+0000","last_active":"2024-09-13T14:14:23.401073+0000","last_peered":"2024-09-13T14:14:23.401073+0000","last_clean":"2024-09-13T14:14:23.401073+0000","last_became_active":"2024-09-13T14:12:26.296739+0000","last_became_peered":"2024-09-13T14:12:26.296739+0000","last_unstale":"2024-09-13T14:14:23.401073+0000","last_undegraded":"2024-09-13T14:14:23.401073+0000","last_fullsized":"2024-09-13T14:14:23.401073+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:14.744825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042041700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397871+0000","last_change":"2024-09-13T14:13:41.131216+0000","last_active":"2024-09-13T14:14:23.397871+0000","last_peered":"2024-09-13T14:14:23.397871+0000","last_clean":"2024-09-13T14:14:23.397871+0000","last_became_active":"2024-09-13T14:13:41.130942+0000","last_became_peered":"2024-09-13T14:13:41.130942+0000","last_unstale":"2024-09-13T14:14:23.397871+0000","last_undegraded":"2024-09-13T14:14:23.397871+0000","last_fullsized":"2024-09-13T14:14:23.397871+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.309907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401035+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:14:23.401035+0000","last_peered":"2024-09-13T14:14:23.401035+0000","last_clean":"2024-09-13T14:14:23.401035+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:14:23.401035+0000","last_undegraded":"2024-09-13T14:14:23.401035+0000","last_fullsized":"2024-09-13T14:14:23.401035+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433692+0000","last_change":"2024-09-13T14:12:31.171913+0000","last_active":"2024-09-13T14:14:23.433692+0000","last_peered":"2024-09-13T14:14:23.433692+0000","last_clean":"2024-09-13T14:14:23.433692+0000","last_became_active":"2024-09-13T14:12:26.551516+0000","last_became_peered":"2024-09-13T14:12:26.551516+0000","last_unstale":"2024-09-13T14:14:23.433692+0000","last_undegraded":"2024-09-13T14:14:23.433692+0000","last_fullsized":"2024-09-13T14:14:23.433692+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.648647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029229000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403317+0000","last_change":"2024-09-13T14:13:41.132884+0000","last_active":"2024-09-13T14:14:23.403317+0000","last_peered":"2024-09-13T14:14:23.403317+0000","last_clean":"2024-09-13T14:14:23.403317+0000","last_became_active":"2024-09-13T14:13:41.132557+0000","last_became_peered":"2024-09-13T14:13:41.132557+0000","last_unstale":"2024-09-13T14:14:23.403317+0000","last_undegraded":"2024-09-13T14:14:23.403317+0000","last_fullsized":"2024-09-13T14:14:23.403317+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:01.648275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403621+0000","last_change":"2024-09-13T14:13:42.150182+0000","last_active":"2024-09-13T14:14:23.403621+0000","last_peered":"2024-09-13T14:14:23.403621+0000","last_clean":"2024-09-13T14:14:23.403621+0000","last_became_active":"2024-09-13T14:13:42.149891+0000","last_became_peered":"2024-09-13T14:13:42.149891+0000","last_unstale":"2024-09-13T14:14:23.403621+0000","last_undegraded":"2024-09-13T14:14:23.403621+0000","last_fullsized":"2024-09-13T14:14:23.403621+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:28.793763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398831+0000","last_change":"2024-09-13T14:12:31.081764+0000","last_active":"2024-09-13T14:14:23.398831+0000","last_peered":"2024-09-13T14:14:23.398831+0000","last_clean":"2024-09-13T14:14:23.398831+0000","last_became_active":"2024-09-13T14:12:26.551472+0000","last_became_peered":"2024-09-13T14:12:26.551472+0000","last_unstale":"2024-09-13T14:14:23.398831+0000","last_undegraded":"2024-09-13T14:14:23.398831+0000","last_fullsized":"2024-09-13T14:14:23.398831+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:04:11.250981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039453399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400649+0000","last_change":"2024-09-13T14:13:41.127892+0000","last_active":"2024-09-13T14:14:23.400649+0000","last_peered":"2024-09-13T14:14:23.400649+0000","last_clean":"2024-09-13T14:14:23.400649+0000","last_became_active":"2024-09-13T14:13:41.127621+0000","last_became_peered":"2024-09-13T14:13:41.127621+0000","last_unstale":"2024-09-13T14:14:23.400649+0000","last_undegraded":"2024-09-13T14:14:23.400649+0000","last_fullsized":"2024-09-13T14:14:23.400649+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:53.454219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.419344+0000","last_change":"2024-09-13T14:14:24.419344+0000","last_active":"2024-09-13T14:14:24.419344+0000","last_peered":"2024-09-13T14:14:24.419344+0000","last_clean":"2024-09-13T14:14:24.419344+0000","last_became_active":"2024-09-13T14:14:24.418981+0000","last_became_peered":"2024-09-13T14:14:24.418981+0000","last_unstale":"2024-09-13T14:14:24.419344+0000","last_undegraded":"2024-09-13T14:14:24.419344+0000","last_fullsized":"2024-09-13T14:14:24.419344+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:36.496394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":28,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401133+0000","last_change":"2024-09-13T14:12:31.086545+0000","last_active":"2024-09-13T14:14:23.401133+0000","last_peered":"2024-09-13T14:14:23.401133+0000","last_clean":"2024-09-13T14:14:23.401133+0000","last_became_active":"2024-09-13T14:12:26.297541+0000","last_became_peered":"2024-09-13T14:12:26.297541+0000","last_unstale":"2024-09-13T14:14:23.401133+0000","last_undegraded":"2024-09-13T14:14:23.401133+0000","last_fullsized":"2024-09-13T14:14:23.401133+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:54.129876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001036726,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398780+0000","last_change":"2024-09-13T14:13:41.126747+0000","last_active":"2024-09-13T14:14:23.398780+0000","last_peered":"2024-09-13T14:14:23.398780+0000","last_clean":"2024-09-13T14:14:23.398780+0000","last_became_active":"2024-09-13T14:13:41.126465+0000","last_became_peered":"2024-09-13T14:13:41.126465+0000","last_unstale":"2024-09-13T14:14:23.398780+0000","last_undegraded":"2024-09-13T14:14:23.398780+0000","last_fullsized":"2024-09-13T14:14:23.398780+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:57.216756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400984+0000","last_change":"2024-09-13T14:13:41.150310+0000","last_active":"2024-09-13T14:14:23.400984+0000","last_peered":"2024-09-13T14:14:23.400984+0000","last_clean":"2024-09-13T14:14:23.400984+0000","last_became_active":"2024-09-13T14:13:41.150041+0000","last_became_peered":"2024-09-13T14:13:41.150041+0000","last_unstale":"2024-09-13T14:14:23.400984+0000","last_undegraded":"2024-09-13T14:14:23.400984+0000","last_fullsized":"2024-09-13T14:14:23.400984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:29.604979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401829+0000","last_change":"2024-09-13T14:13:41.139508+0000","last_active":"2024-09-13T14:14:23.401829+0000","last_peered":"2024-09-13T14:14:23.401829+0000","last_clean":"2024-09-13T14:14:23.401829+0000","last_became_active":"2024-09-13T14:13:41.139227+0000","last_became_peered":"2024-09-13T14:13:41.139227+0000","last_unstale":"2024-09-13T14:14:23.401829+0000","last_undegraded":"2024-09-13T14:14:23.401829+0000","last_fullsized":"2024-09-13T14:14:23.401829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:34.799511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401102+0000","last_change":"2024-09-13T14:12:31.081654+0000","last_active":"2024-09-13T14:14:23.401102+0000","last_peered":"2024-09-13T14:14:23.401102+0000","last_clean":"2024-09-13T14:14:23.401102+0000","last_became_active":"2024-09-13T14:12:26.551473+0000","last_became_peered":"2024-09-13T14:12:26.551473+0000","last_unstale":"2024-09-13T14:14:23.401102+0000","last_undegraded":"2024-09-13T14:14:23.401102+0000","last_fullsized":"2024-09-13T14:14:23.401102+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:42:25.018815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000346795,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400654+0000","last_change":"2024-09-13T14:13:41.149515+0000","last_active":"2024-09-13T14:14:23.400654+0000","last_peered":"2024-09-13T14:14:23.400654+0000","last_clean":"2024-09-13T14:14:23.400654+0000","last_became_active":"2024-09-13T14:13:41.149103+0000","last_became_peered":"2024-09-13T14:13:41.149103+0000","last_unstale":"2024-09-13T14:14:23.400654+0000","last_undegraded":"2024-09-13T14:14:23.400654+0000","last_fullsized":"2024-09-13T14:14:23.400654+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:51.077174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401049+0000","last_change":"2024-09-13T14:13:41.129848+0000","last_active":"2024-09-13T14:14:23.401049+0000","last_peered":"2024-09-13T14:14:23.401049+0000","last_clean":"2024-09-13T14:14:23.401049+0000","last_became_active":"2024-09-13T14:13:41.129548+0000","last_became_peered":"2024-09-13T14:13:41.129548+0000","last_unstale":"2024-09-13T14:14:23.401049+0000","last_undegraded":"2024-09-13T14:14:23.401049+0000","last_fullsized":"2024-09-13T14:14:23.401049+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:40:26.100813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433967+0000","last_change":"2024-09-13T14:12:31.172062+0000","last_active":"2024-09-13T14:14:23.433967+0000","last_peered":"2024-09-13T14:14:23.433967+0000","last_clean":"2024-09-13T14:14:23.433967+0000","last_became_active":"2024-09-13T14:12:26.816843+0000","last_became_peered":"2024-09-13T14:12:26.816843+0000","last_unstale":"2024-09-13T14:14:23.433967+0000","last_undegraded":"2024-09-13T14:14:23.433967+0000","last_fullsized":"2024-09-13T14:14:23.433967+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:53.145885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343452,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403533+0000","last_change":"2024-09-13T14:13:42.150968+0000","last_active":"2024-09-13T14:14:23.403533+0000","last_peered":"2024-09-13T14:14:23.403533+0000","last_clean":"2024-09-13T14:14:23.403533+0000","last_became_active":"2024-09-13T14:13:42.150702+0000","last_became_peered":"2024-09-13T14:13:42.150702+0000","last_unstale":"2024-09-13T14:14:23.403533+0000","last_undegraded":"2024-09-13T14:14:23.403533+0000","last_fullsized":"2024-09-13T14:14:23.403533+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:48.691675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398376+0000","last_change":"2024-09-13T14:13:41.146075+0000","last_active":"2024-09-13T14:14:23.398376+0000","last_peered":"2024-09-13T14:14:23.398376+0000","last_clean":"2024-09-13T14:14:23.398376+0000","last_became_active":"2024-09-13T14:13:41.145706+0000","last_became_peered":"2024-09-13T14:13:41.145706+0000","last_unstale":"2024-09-13T14:14:23.398376+0000","last_undegraded":"2024-09-13T14:14:23.398376+0000","last_fullsized":"2024-09-13T14:14:23.398376+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:25.189062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401294+0000","last_change":"2024-09-13T14:12:31.081944+0000","last_active":"2024-09-13T14:14:23.401294+0000","last_peered":"2024-09-13T14:14:23.401294+0000","last_clean":"2024-09-13T14:14:23.401294+0000","last_became_active":"2024-09-13T14:12:26.816862+0000","last_became_peered":"2024-09-13T14:12:26.816862+0000","last_unstale":"2024-09-13T14:14:23.401294+0000","last_undegraded":"2024-09-13T14:14:23.401294+0000","last_fullsized":"2024-09-13T14:14:23.401294+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:43.703026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062990400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397651+0000","last_change":"2024-09-13T14:13:42.152844+0000","last_active":"2024-09-13T14:14:23.397651+0000","last_peered":"2024-09-13T14:14:23.397651+0000","last_clean":"2024-09-13T14:14:23.397651+0000","last_became_active":"2024-09-13T14:13:42.152016+0000","last_became_peered":"2024-09-13T14:13:42.152016+0000","last_unstale":"2024-09-13T14:14:23.397651+0000","last_undegraded":"2024-09-13T14:14:23.397651+0000","last_fullsized":"2024-09-13T14:14:23.397651+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:58.105884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403723+0000","last_change":"2024-09-13T14:13:41.126928+0000","last_active":"2024-09-13T14:14:23.403723+0000","last_peered":"2024-09-13T14:14:23.403723+0000","last_clean":"2024-09-13T14:14:23.403723+0000","last_became_active":"2024-09-13T14:13:41.126243+0000","last_became_peered":"2024-09-13T14:13:41.126243+0000","last_unstale":"2024-09-13T14:14:23.403723+0000","last_undegraded":"2024-09-13T14:14:23.403723+0000","last_fullsized":"2024-09-13T14:14:23.403723+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:48.937352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401570+0000","last_change":"2024-09-13T14:12:31.087292+0000","last_active":"2024-09-13T14:14:23.401570+0000","last_peered":"2024-09-13T14:14:23.401570+0000","last_clean":"2024-09-13T14:14:23.401570+0000","last_became_active":"2024-09-13T14:12:26.296590+0000","last_became_peered":"2024-09-13T14:12:26.296590+0000","last_unstale":"2024-09-13T14:14:23.401570+0000","last_undegraded":"2024-09-13T14:14:23.401570+0000","last_fullsized":"2024-09-13T14:14:23.401570+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:02.313429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026736199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398216+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:14:23.398216+0000","last_peered":"2024-09-13T14:14:23.398216+0000","last_clean":"2024-09-13T14:14:23.398216+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:14:23.398216+0000","last_undegraded":"2024-09-13T14:14:23.398216+0000","last_fullsized":"2024-09-13T14:14:23.398216+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398698+0000","last_change":"2024-09-13T14:13:41.150310+0000","last_active":"2024-09-13T14:14:23.398698+0000","last_peered":"2024-09-13T14:14:23.398698+0000","last_clean":"2024-09-13T14:14:23.398698+0000","last_became_active":"2024-09-13T14:13:41.150035+0000","last_became_peered":"2024-09-13T14:13:41.150035+0000","last_unstale":"2024-09-13T14:14:23.398698+0000","last_undegraded":"2024-09-13T14:14:23.398698+0000","last_fullsized":"2024-09-13T14:14:23.398698+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:26.164328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398490+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:14:23.398490+0000","last_peered":"2024-09-13T14:14:23.398490+0000","last_clean":"2024-09-13T14:14:23.398490+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:14:23.398490+0000","last_undegraded":"2024-09-13T14:14:23.398490+0000","last_fullsized":"2024-09-13T14:14:23.398490+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400642+0000","last_change":"2024-09-13T14:13:41.147747+0000","last_active":"2024-09-13T14:14:23.400642+0000","last_peered":"2024-09-13T14:14:23.400642+0000","last_clean":"2024-09-13T14:14:23.400642+0000","last_became_active":"2024-09-13T14:13:41.147209+0000","last_became_peered":"2024-09-13T14:13:41.147209+0000","last_unstale":"2024-09-13T14:14:23.400642+0000","last_undegraded":"2024-09-13T14:14:23.400642+0000","last_fullsized":"2024-09-13T14:14:23.400642+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:55.514702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433864+0000","last_change":"2024-09-13T14:13:41.146093+0000","last_active":"2024-09-13T14:14:23.433864+0000","last_peered":"2024-09-13T14:14:23.433864+0000","last_clean":"2024-09-13T14:14:23.433864+0000","last_became_active":"2024-09-13T14:13:41.145816+0000","last_became_peered":"2024-09-13T14:13:41.145816+0000","last_unstale":"2024-09-13T14:14:23.433864+0000","last_undegraded":"2024-09-13T14:14:23.433864+0000","last_fullsized":"2024-09-13T14:14:23.433864+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:19.263045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401772+0000","last_change":"2024-09-13T14:13:41.133453+0000","last_active":"2024-09-13T14:14:23.401772+0000","last_peered":"2024-09-13T14:14:23.401772+0000","last_clean":"2024-09-13T14:14:23.401772+0000","last_became_active":"2024-09-13T14:13:41.133209+0000","last_became_peered":"2024-09-13T14:13:41.133209+0000","last_unstale":"2024-09-13T14:14:23.401772+0000","last_undegraded":"2024-09-13T14:14:23.401772+0000","last_fullsized":"2024-09-13T14:14:23.401772+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:33.498683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397649+0000","last_change":"2024-09-13T14:13:42.489086+0000","last_active":"2024-09-13T14:14:23.397649+0000","last_peered":"2024-09-13T14:14:23.397649+0000","last_clean":"2024-09-13T14:14:23.397649+0000","last_became_active":"2024-09-13T14:13:42.488762+0000","last_became_peered":"2024-09-13T14:13:42.488762+0000","last_unstale":"2024-09-13T14:14:23.397649+0000","last_undegraded":"2024-09-13T14:14:23.397649+0000","last_fullsized":"2024-09-13T14:14:23.397649+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:59:18.584820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403921+0000","last_change":"2024-09-13T14:13:41.136346+0000","last_active":"2024-09-13T14:14:23.403921+0000","last_peered":"2024-09-13T14:14:23.403921+0000","last_clean":"2024-09-13T14:14:23.403921+0000","last_became_active":"2024-09-13T14:13:41.135422+0000","last_became_peered":"2024-09-13T14:13:41.135422+0000","last_unstale":"2024-09-13T14:14:23.403921+0000","last_undegraded":"2024-09-13T14:14:23.403921+0000","last_fullsized":"2024-09-13T14:14:23.403921+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:11.340917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397625+0000","last_change":"2024-09-13T14:13:42.489075+0000","last_active":"2024-09-13T14:14:23.397625+0000","last_peered":"2024-09-13T14:14:23.397625+0000","last_clean":"2024-09-13T14:14:23.397625+0000","last_became_active":"2024-09-13T14:13:42.488738+0000","last_became_peered":"2024-09-13T14:13:42.488738+0000","last_unstale":"2024-09-13T14:14:23.397625+0000","last_undegraded":"2024-09-13T14:14:23.397625+0000","last_fullsized":"2024-09-13T14:14:23.397625+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:11.076752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401489+0000","last_change":"2024-09-13T14:13:41.150482+0000","last_active":"2024-09-13T14:14:23.401489+0000","last_peered":"2024-09-13T14:14:23.401489+0000","last_clean":"2024-09-13T14:14:23.401489+0000","last_became_active":"2024-09-13T14:13:41.150196+0000","last_became_peered":"2024-09-13T14:13:41.150196+0000","last_unstale":"2024-09-13T14:14:23.401489+0000","last_undegraded":"2024-09-13T14:14:23.401489+0000","last_fullsized":"2024-09-13T14:14:23.401489+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:31.060006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397924+0000","last_change":"2024-09-13T14:13:41.141497+0000","last_active":"2024-09-13T14:14:23.397924+0000","last_peered":"2024-09-13T14:14:23.397924+0000","last_clean":"2024-09-13T14:14:23.397924+0000","last_became_active":"2024-09-13T14:13:41.139267+0000","last_became_peered":"2024-09-13T14:13:41.139267+0000","last_unstale":"2024-09-13T14:14:23.397924+0000","last_undegraded":"2024-09-13T14:14:23.397924+0000","last_fullsized":"2024-09-13T14:14:23.397924+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:40.482904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403614+0000","last_change":"2024-09-13T14:13:42.150796+0000","last_active":"2024-09-13T14:14:23.403614+0000","last_peered":"2024-09-13T14:14:23.403614+0000","last_clean":"2024-09-13T14:14:23.403614+0000","last_became_active":"2024-09-13T14:13:42.150512+0000","last_became_peered":"2024-09-13T14:13:42.150512+0000","last_unstale":"2024-09-13T14:14:23.403614+0000","last_undegraded":"2024-09-13T14:14:23.403614+0000","last_fullsized":"2024-09-13T14:14:23.403614+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:11.912670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400500+0000","last_change":"2024-09-13T14:13:41.146409+0000","last_active":"2024-09-13T14:14:23.400500+0000","last_peered":"2024-09-13T14:14:23.400500+0000","last_clean":"2024-09-13T14:14:23.400500+0000","last_became_active":"2024-09-13T14:13:41.146125+0000","last_became_peered":"2024-09-13T14:13:41.146125+0000","last_unstale":"2024-09-13T14:14:23.400500+0000","last_undegraded":"2024-09-13T14:14:23.400500+0000","last_fullsized":"2024-09-13T14:14:23.400500+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:28.588056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398171+0000","last_change":"2024-09-13T14:13:42.607992+0000","last_active":"2024-09-13T14:14:23.398171+0000","last_peered":"2024-09-13T14:14:23.398171+0000","last_clean":"2024-09-13T14:14:23.398171+0000","last_became_active":"2024-09-13T14:13:42.607575+0000","last_became_peered":"2024-09-13T14:13:42.607575+0000","last_unstale":"2024-09-13T14:14:23.398171+0000","last_undegraded":"2024-09-13T14:14:23.398171+0000","last_fullsized":"2024-09-13T14:14:23.398171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:53.106609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398430+0000","last_change":"2024-09-13T14:13:41.130505+0000","last_active":"2024-09-13T14:14:23.398430+0000","last_peered":"2024-09-13T14:14:23.398430+0000","last_clean":"2024-09-13T14:14:23.398430+0000","last_became_active":"2024-09-13T14:13:41.130120+0000","last_became_peered":"2024-09-13T14:13:41.130120+0000","last_unstale":"2024-09-13T14:14:23.398430+0000","last_undegraded":"2024-09-13T14:14:23.398430+0000","last_fullsized":"2024-09-13T14:14:23.398430+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:50.637901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398825+0000","last_change":"2024-09-13T14:13:41.147138+0000","last_active":"2024-09-13T14:14:23.398825+0000","last_peered":"2024-09-13T14:14:23.398825+0000","last_clean":"2024-09-13T14:14:23.398825+0000","last_became_active":"2024-09-13T14:13:41.146828+0000","last_became_peered":"2024-09-13T14:13:41.146828+0000","last_unstale":"2024-09-13T14:14:23.398825+0000","last_undegraded":"2024-09-13T14:14:23.398825+0000","last_fullsized":"2024-09-13T14:14:23.398825+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:26.095253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403628+0000","last_change":"2024-09-13T14:13:41.130035+0000","last_active":"2024-09-13T14:14:23.403628+0000","last_peered":"2024-09-13T14:14:23.403628+0000","last_clean":"2024-09-13T14:14:23.403628+0000","last_became_active":"2024-09-13T14:13:41.129732+0000","last_became_peered":"2024-09-13T14:13:41.129732+0000","last_unstale":"2024-09-13T14:14:23.403628+0000","last_undegraded":"2024-09-13T14:14:23.403628+0000","last_fullsized":"2024-09-13T14:14:23.403628+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:30.425166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398640+0000","last_change":"2024-09-13T14:13:41.150436+0000","last_active":"2024-09-13T14:14:23.398640+0000","last_peered":"2024-09-13T14:14:23.398640+0000","last_clean":"2024-09-13T14:14:23.398640+0000","last_became_active":"2024-09-13T14:13:41.150174+0000","last_became_peered":"2024-09-13T14:13:41.150174+0000","last_unstale":"2024-09-13T14:14:23.398640+0000","last_undegraded":"2024-09-13T14:14:23.398640+0000","last_fullsized":"2024-09-13T14:14:23.398640+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:16.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401309+0000","last_change":"2024-09-13T14:13:41.131340+0000","last_active":"2024-09-13T14:14:23.401309+0000","last_peered":"2024-09-13T14:14:23.401309+0000","last_clean":"2024-09-13T14:14:23.401309+0000","last_became_active":"2024-09-13T14:13:41.130996+0000","last_became_peered":"2024-09-13T14:13:41.130996+0000","last_unstale":"2024-09-13T14:14:23.401309+0000","last_undegraded":"2024-09-13T14:14:23.401309+0000","last_fullsized":"2024-09-13T14:14:23.401309+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:09.294099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398339+0000","last_change":"2024-09-13T14:13:41.149509+0000","last_active":"2024-09-13T14:14:23.398339+0000","last_peered":"2024-09-13T14:14:23.398339+0000","last_clean":"2024-09-13T14:14:23.398339+0000","last_became_active":"2024-09-13T14:13:41.149217+0000","last_became_peered":"2024-09-13T14:13:41.149217+0000","last_unstale":"2024-09-13T14:14:23.398339+0000","last_undegraded":"2024-09-13T14:14:23.398339+0000","last_fullsized":"2024-09-13T14:14:23.398339+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:59.273576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401741+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:14:23.401741+0000","last_peered":"2024-09-13T14:14:23.401741+0000","last_clean":"2024-09-13T14:14:23.401741+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:14:23.401741+0000","last_undegraded":"2024-09-13T14:14:23.401741+0000","last_fullsized":"2024-09-13T14:14:23.401741+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397829+0000","last_change":"2024-09-13T14:13:42.151860+0000","last_active":"2024-09-13T14:14:23.397829+0000","last_peered":"2024-09-13T14:14:23.397829+0000","last_clean":"2024-09-13T14:14:23.397829+0000","last_became_active":"2024-09-13T14:13:42.151429+0000","last_became_peered":"2024-09-13T14:13:42.151429+0000","last_unstale":"2024-09-13T14:14:23.397829+0000","last_undegraded":"2024-09-13T14:14:23.397829+0000","last_fullsized":"2024-09-13T14:14:23.397829+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:24:41.330481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398351+0000","last_change":"2024-09-13T14:13:41.128835+0000","last_active":"2024-09-13T14:14:23.398351+0000","last_peered":"2024-09-13T14:14:23.398351+0000","last_clean":"2024-09-13T14:14:23.398351+0000","last_became_active":"2024-09-13T14:13:41.128440+0000","last_became_peered":"2024-09-13T14:13:41.128440+0000","last_unstale":"2024-09-13T14:14:23.398351+0000","last_undegraded":"2024-09-13T14:14:23.398351+0000","last_fullsized":"2024-09-13T14:14:23.398351+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:06.588300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.758890+0000","last_change":"2024-09-13T14:14:24.758890+0000","last_active":"2024-09-13T14:14:24.758890+0000","last_peered":"2024-09-13T14:14:24.758890+0000","last_clean":"2024-09-13T14:14:24.758890+0000","last_became_active":"2024-09-13T14:14:24.758588+0000","last_became_peered":"2024-09-13T14:14:24.758588+0000","last_unstale":"2024-09-13T14:14:24.758890+0000","last_undegraded":"2024-09-13T14:14:24.758890+0000","last_fullsized":"2024-09-13T14:14:24.758890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:25.781156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403903+0000","last_change":"2024-09-13T14:13:41.136986+0000","last_active":"2024-09-13T14:14:23.403903+0000","last_peered":"2024-09-13T14:14:23.403903+0000","last_clean":"2024-09-13T14:14:23.403903+0000","last_became_active":"2024-09-13T14:13:41.136609+0000","last_became_peered":"2024-09-13T14:13:41.136609+0000","last_unstale":"2024-09-13T14:14:23.403903+0000","last_undegraded":"2024-09-13T14:14:23.403903+0000","last_fullsized":"2024-09-13T14:14:23.403903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:46.403461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":888643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.53600000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78300000000000003,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68700000000000006,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.57199999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66700000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58399999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68899999999999995,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.503},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71599999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.48099999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.41099999999999998,"15min":0.40500000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.374}]},{"osd":5,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.432,"15min":0.432},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.39800000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.54500000000000004}]}]},{"osd":6,"up_from":43,"seq":184683593769,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1248,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1277952,"data_stored":887957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.504,"15min":0.49399999999999999},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72099999999999997,"5min":1.083,"15min":1.083},"last":0.64500000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.497,"15min":0.48799999999999999},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72099999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52200000000000002,"15min":0.51600000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54900000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.42299999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53000000000000003,"15min":0.52200000000000002},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56999999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66900000000000004,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76100000000000001,"5min":1.117,"15min":1.117},"last":0.48399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.41199999999999998,"15min":0.39400000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.499}]},{"osd":5,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.441,"15min":0.42099999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70899999999999996,"5min":0.754,"15min":0.754},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.45300000000000001,"15min":0.432},"min":{"1min":0.30599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68999999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46800000000000003}]},{"osd":7,"last update":"Fri Sep 13 14:13:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.503,"15min":0.495},"min":{"1min":0.36699999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.58099999999999996}]}]},{"osd":5,"up_from":37,"seq":158913789999,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":675840,"data_stored":296536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.56499999999999995},"min":{"1min":0.42399999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.66800000000000004,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54700000000000004,"15min":0.56499999999999995},"min":{"1min":0.40999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72099999999999997,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.40300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73199999999999998,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.44800000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72699999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.44800000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55200000000000005,"15min":0.56399999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.56000000000000005,"15min":0.57299999999999995},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56699999999999995,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.66400000000000003,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.47899999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.65700000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59499999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.41399999999999998,"15min":0.39600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.41599999999999998,"15min":0.39600000000000002},"min":{"1min":0.36899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.432,"15min":0.42499999999999999},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.43099999999999999,"15min":0.41899999999999998},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68600000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.64800000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47599999999999998,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.47899999999999998,"15min":0.47199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.58099999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":298205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.51300000000000001,"15min":0.52700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64000000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.375},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.63500000000000001,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.39800000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.50800000000000001,"15min":0.51800000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65300000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.505,"15min":0.51700000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68600000000000005,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.48299999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.50800000000000001,"15min":0.51100000000000001},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":0.69799999999999995,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51000000000000001,"15min":0.51400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68899999999999995,"5min":0.879,"15min":0.879},"last":0.46200000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.753},"last":0.496},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54300000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:14:10 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.36899999999999999,"15min":0.34200000000000003},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.375,"15min":0.34899999999999998},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63200000000000001,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.53200000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.40799999999999997,"15min":0.39300000000000002},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66000000000000003,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.46700000000000003,"15min":0.45500000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.71599999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.46899999999999997,"15min":0.46500000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.755,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.497,"5min":0.48099999999999998,"15min":0.47799999999999998},"min":{"1min":0.30199999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.58099999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":675840,"data_stored":296597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.376,"15min":0.377},"min":{"1min":0.21299999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40400000000000003,"15min":0.40200000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.45700000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.25,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.372},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.28799999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45400000000000001,"15min":0.46100000000000002},"min":{"1min":0.26800000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.51800000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:13:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83099999999999996,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:14:08 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.623,"15min":0.61899999999999999},"min":{"1min":0.45800000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.61299999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:14:35 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42899999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.63100000000000001,"15min":0.626},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59999999999999998,"15min":0.59399999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59899999999999998},"min":{"1min":0.49399999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.53900000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":679936,"data_stored":297111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.36799999999999999,"15min":0.36399999999999999},"min":{"1min":0.247,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.52600000000000002,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.379,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.26200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.374,"15min":0.374},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.248},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.58399999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.29499999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:14:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.439,"15min":0.45600000000000002},"min":{"1min":0.30599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.315},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.42499999999999999,"15min":0.44500000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:13:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.60399999999999998,"15min":0.59399999999999997},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78500000000000003,"5min":1.024,"15min":1.024},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71499999999999997,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.65500000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80900000000000005,"5min":1.101,"15min":1.101},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77200000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:14:35 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61399999999999999,"15min":0.63700000000000001},"min":{"1min":0.39300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80900000000000005,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.622,"15min":0.64500000000000002},"min":{"1min":0.45100000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.85199999999999998,"5min":1.04,"15min":1.04},"last":0.60299999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.625,"15min":0.63800000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.746,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60599999999999998,"15min":0.61499999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77500000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.63200000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.373,"15min":0.35299999999999998},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.47399999999999998,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.379,"5min":0.376,"15min":0.35799999999999998},"min":{"1min":0.30499999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.46000000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33300000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.38200000000000001,"15min":0.38500000000000001},"min":{"1min":0.25700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54500000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38700000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.42199999999999999,"15min":0.435},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.51000000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.37},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.435,"15min":0.44900000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59699999999999998,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.30399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.61899999999999999,"15min":0.61399999999999999},"min":{"1min":0.45800000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.47699999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.53500000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.63600000000000001,"15min":0.64100000000000001},"min":{"1min":0.433,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0069999999999999,"5min":1.129,"15min":1.129},"last":0.51000000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.42899999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76700000000000002,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63800000000000001,"15min":0.64600000000000002},"min":{"1min":0.51100000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81299999999999994,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.48099999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.90500000000000003,"5min":1.252,"15min":1.252},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.97099999999999997,"5min":1.415,"15min":1.415},"last":0.497}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298600,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446552,"statfs":{"total":95995035648,"available":95689269248,"internally_reserved":0,"allocated":1269760,"data_stored":887152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:13:44 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.34499999999999997},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.374,"15min":0.35399999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:14:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.25600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.42399999999999999,"15min":0.40799999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.66100000000000003,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.28699999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:14:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.371},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.34100000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:13:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60599999999999998,"15min":0.59799999999999998},"min":{"1min":0.39900000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.52000000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.436,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.84399999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.629,"15min":0.626},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.61899999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:14:36 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.61199999999999999,"15min":0.60799999999999998},"min":{"1min":0.47899999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.629,"15min":0.624},"min":{"1min":0.47099999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.68600000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.61299999999999999,"15min":0.60599999999999998},"min":{"1min":0.40300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.63200000000000001,"15min":0.625},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:14:41.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:14:41.477 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[29399]: from='client.24653 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:41.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[38255]: from='client.24653 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:41 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:41 smithi189 ceph-mon[33294]: from='client.24653 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:41 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:41 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:41 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:42.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:42 smithi017 ceph-mon[29399]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:42.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:42 smithi017 ceph-mon[38255]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:42 smithi189 ceph-mon[33294]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:14:43.345 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:14:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-13T14:14:43.315327+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385928,"kb_used_data":7076,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575288,"statfs":{"total":767960285184,"available":765517094912,"internally_reserved":0,"allocated":7245824,"data_stored":4148727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002759"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398058+0000","last_change":"2024-09-13T14:13:42.151465+0000","last_active":"2024-09-13T14:14:23.398058+0000","last_peered":"2024-09-13T14:14:23.398058+0000","last_clean":"2024-09-13T14:14:23.398058+0000","last_became_active":"2024-09-13T14:13:42.151206+0000","last_became_peered":"2024-09-13T14:13:42.151206+0000","last_unstale":"2024-09-13T14:14:23.398058+0000","last_undegraded":"2024-09-13T14:14:23.398058+0000","last_fullsized":"2024-09-13T14:14:23.398058+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:14.848155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433717+0000","last_change":"2024-09-13T14:13:41.136356+0000","last_active":"2024-09-13T14:14:23.433717+0000","last_peered":"2024-09-13T14:14:23.433717+0000","last_clean":"2024-09-13T14:14:23.433717+0000","last_became_active":"2024-09-13T14:13:41.136101+0000","last_became_peered":"2024-09-13T14:13:41.136101+0000","last_unstale":"2024-09-13T14:14:23.433717+0000","last_undegraded":"2024-09-13T14:14:23.433717+0000","last_fullsized":"2024-09-13T14:14:23.433717+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:44.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401729+0000","last_change":"2024-09-13T14:13:42.150106+0000","last_active":"2024-09-13T14:14:23.401729+0000","last_peered":"2024-09-13T14:14:23.401729+0000","last_clean":"2024-09-13T14:14:23.401729+0000","last_became_active":"2024-09-13T14:13:42.149808+0000","last_became_peered":"2024-09-13T14:13:42.149808+0000","last_unstale":"2024-09-13T14:14:23.401729+0000","last_undegraded":"2024-09-13T14:14:23.401729+0000","last_fullsized":"2024-09-13T14:14:23.401729+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:21.087262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403365+0000","last_change":"2024-09-13T14:13:41.131071+0000","last_active":"2024-09-13T14:14:23.403365+0000","last_peered":"2024-09-13T14:14:23.403365+0000","last_clean":"2024-09-13T14:14:23.403365+0000","last_became_active":"2024-09-13T14:13:41.130771+0000","last_became_peered":"2024-09-13T14:13:41.130771+0000","last_unstale":"2024-09-13T14:14:23.403365+0000","last_undegraded":"2024-09-13T14:14:23.403365+0000","last_fullsized":"2024-09-13T14:14:23.403365+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:32.111331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403708+0000","last_change":"2024-09-13T14:13:42.149841+0000","last_active":"2024-09-13T14:14:23.403708+0000","last_peered":"2024-09-13T14:14:23.403708+0000","last_clean":"2024-09-13T14:14:23.403708+0000","last_became_active":"2024-09-13T14:13:42.149564+0000","last_became_peered":"2024-09-13T14:13:42.149564+0000","last_unstale":"2024-09-13T14:14:23.403708+0000","last_undegraded":"2024-09-13T14:14:23.403708+0000","last_fullsized":"2024-09-13T14:14:23.403708+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:29:58.753171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433655+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:14:23.433655+0000","last_peered":"2024-09-13T14:14:23.433655+0000","last_clean":"2024-09-13T14:14:23.433655+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:14:23.433655+0000","last_undegraded":"2024-09-13T14:14:23.433655+0000","last_fullsized":"2024-09-13T14:14:23.433655+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398171+0000","last_change":"2024-09-13T14:13:42.151443+0000","last_active":"2024-09-13T14:14:23.398171+0000","last_peered":"2024-09-13T14:14:23.398171+0000","last_clean":"2024-09-13T14:14:23.398171+0000","last_became_active":"2024-09-13T14:13:42.151156+0000","last_became_peered":"2024-09-13T14:13:42.151156+0000","last_unstale":"2024-09-13T14:14:23.398171+0000","last_undegraded":"2024-09-13T14:14:23.398171+0000","last_fullsized":"2024-09-13T14:14:23.398171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:18.895996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401095+0000","last_change":"2024-09-13T14:13:41.130289+0000","last_active":"2024-09-13T14:14:23.401095+0000","last_peered":"2024-09-13T14:14:23.401095+0000","last_clean":"2024-09-13T14:14:23.401095+0000","last_became_active":"2024-09-13T14:13:41.129333+0000","last_became_peered":"2024-09-13T14:13:41.129333+0000","last_unstale":"2024-09-13T14:14:23.401095+0000","last_undegraded":"2024-09-13T14:14:23.401095+0000","last_fullsized":"2024-09-13T14:14:23.401095+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:16.787146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.434023+0000","last_change":"2024-09-13T14:13:41.132344+0000","last_active":"2024-09-13T14:14:23.434023+0000","last_peered":"2024-09-13T14:14:23.434023+0000","last_clean":"2024-09-13T14:14:23.434023+0000","last_became_active":"2024-09-13T14:13:41.132064+0000","last_became_peered":"2024-09-13T14:13:41.132064+0000","last_unstale":"2024-09-13T14:14:23.434023+0000","last_undegraded":"2024-09-13T14:14:23.434023+0000","last_fullsized":"2024-09-13T14:14:23.434023+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:19:00.797682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400890+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:14:23.400890+0000","last_peered":"2024-09-13T14:14:23.400890+0000","last_clean":"2024-09-13T14:14:23.400890+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:14:23.400890+0000","last_undegraded":"2024-09-13T14:14:23.400890+0000","last_fullsized":"2024-09-13T14:14:23.400890+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.434017+0000","last_change":"2024-09-13T14:13:41.130783+0000","last_active":"2024-09-13T14:14:23.434017+0000","last_peered":"2024-09-13T14:14:23.434017+0000","last_clean":"2024-09-13T14:14:23.434017+0000","last_became_active":"2024-09-13T14:13:41.130531+0000","last_became_peered":"2024-09-13T14:13:41.130531+0000","last_unstale":"2024-09-13T14:14:23.434017+0000","last_undegraded":"2024-09-13T14:14:23.434017+0000","last_fullsized":"2024-09-13T14:14:23.434017+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:01.706615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398278+0000","last_change":"2024-09-13T14:13:41.150904+0000","last_active":"2024-09-13T14:14:23.398278+0000","last_peered":"2024-09-13T14:14:23.398278+0000","last_clean":"2024-09-13T14:14:23.398278+0000","last_became_active":"2024-09-13T14:13:41.150184+0000","last_became_peered":"2024-09-13T14:13:41.150184+0000","last_unstale":"2024-09-13T14:14:23.398278+0000","last_undegraded":"2024-09-13T14:14:23.398278+0000","last_fullsized":"2024-09-13T14:14:23.398278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:01.674973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433973+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:14:23.433973+0000","last_peered":"2024-09-13T14:14:23.433973+0000","last_clean":"2024-09-13T14:14:23.433973+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:14:23.433973+0000","last_undegraded":"2024-09-13T14:14:23.433973+0000","last_fullsized":"2024-09-13T14:14:23.433973+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400674+0000","last_change":"2024-09-13T14:13:41.147751+0000","last_active":"2024-09-13T14:14:23.400674+0000","last_peered":"2024-09-13T14:14:23.400674+0000","last_clean":"2024-09-13T14:14:23.400674+0000","last_became_active":"2024-09-13T14:13:41.147448+0000","last_became_peered":"2024-09-13T14:13:41.147448+0000","last_unstale":"2024-09-13T14:14:23.400674+0000","last_undegraded":"2024-09-13T14:14:23.400674+0000","last_fullsized":"2024-09-13T14:14:23.400674+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:50.145254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401397+0000","last_change":"2024-09-13T14:13:41.131269+0000","last_active":"2024-09-13T14:14:23.401397+0000","last_peered":"2024-09-13T14:14:23.401397+0000","last_clean":"2024-09-13T14:14:23.401397+0000","last_became_active":"2024-09-13T14:13:41.131031+0000","last_became_peered":"2024-09-13T14:13:41.131031+0000","last_unstale":"2024-09-13T14:14:23.401397+0000","last_undegraded":"2024-09-13T14:14:23.401397+0000","last_fullsized":"2024-09-13T14:14:23.401397+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:33.641297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403511+0000","last_change":"2024-09-13T14:13:42.607860+0000","last_active":"2024-09-13T14:14:23.403511+0000","last_peered":"2024-09-13T14:14:23.403511+0000","last_clean":"2024-09-13T14:14:23.403511+0000","last_became_active":"2024-09-13T14:13:42.607517+0000","last_became_peered":"2024-09-13T14:13:42.607517+0000","last_unstale":"2024-09-13T14:14:23.403511+0000","last_undegraded":"2024-09-13T14:14:23.403511+0000","last_fullsized":"2024-09-13T14:14:23.403511+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:17:06.998105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398088+0000","last_change":"2024-09-13T14:13:41.148026+0000","last_active":"2024-09-13T14:14:23.398088+0000","last_peered":"2024-09-13T14:14:23.398088+0000","last_clean":"2024-09-13T14:14:23.398088+0000","last_became_active":"2024-09-13T14:13:41.146192+0000","last_became_peered":"2024-09-13T14:13:41.146192+0000","last_unstale":"2024-09-13T14:14:23.398088+0000","last_undegraded":"2024-09-13T14:14:23.398088+0000","last_fullsized":"2024-09-13T14:14:23.398088+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:17.333126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398384+0000","last_change":"2024-09-13T14:13:41.144813+0000","last_active":"2024-09-13T14:14:23.398384+0000","last_peered":"2024-09-13T14:14:23.398384+0000","last_clean":"2024-09-13T14:14:23.398384+0000","last_became_active":"2024-09-13T14:13:41.144542+0000","last_became_peered":"2024-09-13T14:13:41.144542+0000","last_unstale":"2024-09-13T14:14:23.398384+0000","last_undegraded":"2024-09-13T14:14:23.398384+0000","last_fullsized":"2024-09-13T14:14:23.398384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:29.013377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400576+0000","last_change":"2024-09-13T14:13:41.126997+0000","last_active":"2024-09-13T14:14:23.400576+0000","last_peered":"2024-09-13T14:14:23.400576+0000","last_clean":"2024-09-13T14:14:23.400576+0000","last_became_active":"2024-09-13T14:13:41.126723+0000","last_became_peered":"2024-09-13T14:13:41.126723+0000","last_unstale":"2024-09-13T14:14:23.400576+0000","last_undegraded":"2024-09-13T14:14:23.400576+0000","last_fullsized":"2024-09-13T14:14:23.400576+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:06.360290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.419739+0000","last_change":"2024-09-13T14:14:24.419739+0000","last_active":"2024-09-13T14:14:24.419739+0000","last_peered":"2024-09-13T14:14:24.419739+0000","last_clean":"2024-09-13T14:14:24.419739+0000","last_became_active":"2024-09-13T14:14:24.419396+0000","last_became_peered":"2024-09-13T14:14:24.419396+0000","last_unstale":"2024-09-13T14:14:24.419739+0000","last_undegraded":"2024-09-13T14:14:24.419739+0000","last_fullsized":"2024-09-13T14:14:24.419739+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:11.051977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398400+0000","last_change":"2024-09-13T14:13:41.133374+0000","last_active":"2024-09-13T14:14:23.398400+0000","last_peered":"2024-09-13T14:14:23.398400+0000","last_clean":"2024-09-13T14:14:23.398400+0000","last_became_active":"2024-09-13T14:13:41.132989+0000","last_became_peered":"2024-09-13T14:13:41.132989+0000","last_unstale":"2024-09-13T14:14:23.398400+0000","last_undegraded":"2024-09-13T14:14:23.398400+0000","last_fullsized":"2024-09-13T14:14:23.398400+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:57.357479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398589+0000","last_change":"2024-09-13T14:13:41.151154+0000","last_active":"2024-09-13T14:14:23.398589+0000","last_peered":"2024-09-13T14:14:23.398589+0000","last_clean":"2024-09-13T14:14:23.398589+0000","last_became_active":"2024-09-13T14:13:41.150785+0000","last_became_peered":"2024-09-13T14:13:41.150785+0000","last_unstale":"2024-09-13T14:14:23.398589+0000","last_undegraded":"2024-09-13T14:14:23.398589+0000","last_fullsized":"2024-09-13T14:14:23.398589+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:18.759670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397779+0000","last_change":"2024-09-13T14:13:41.139264+0000","last_active":"2024-09-13T14:14:23.397779+0000","last_peered":"2024-09-13T14:14:23.397779+0000","last_clean":"2024-09-13T14:14:23.397779+0000","last_became_active":"2024-09-13T14:13:41.138964+0000","last_became_peered":"2024-09-13T14:13:41.138964+0000","last_unstale":"2024-09-13T14:14:23.397779+0000","last_undegraded":"2024-09-13T14:14:23.397779+0000","last_fullsized":"2024-09-13T14:14:23.397779+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:32:51.533046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401184+0000","last_change":"2024-09-13T14:13:41.153668+0000","last_active":"2024-09-13T14:14:23.401184+0000","last_peered":"2024-09-13T14:14:23.401184+0000","last_clean":"2024-09-13T14:14:23.401184+0000","last_became_active":"2024-09-13T14:13:41.153316+0000","last_became_peered":"2024-09-13T14:13:41.153316+0000","last_unstale":"2024-09-13T14:14:23.401184+0000","last_undegraded":"2024-09-13T14:14:23.401184+0000","last_fullsized":"2024-09-13T14:14:23.401184+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:30:23.303445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"53'79","reported_seq":56,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433730+0000","last_change":"2024-09-13T14:11:54.779008+0000","last_active":"2024-09-13T14:14:23.433730+0000","last_peered":"2024-09-13T14:14:23.433730+0000","last_clean":"2024-09-13T14:14:23.433730+0000","last_became_active":"2024-09-13T14:11:54.772279+0000","last_became_peered":"2024-09-13T14:11:54.772279+0000","last_unstale":"2024-09-13T14:14:23.433730+0000","last_undegraded":"2024-09-13T14:14:23.433730+0000","last_fullsized":"2024-09-13T14:14:23.433730+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:02.973115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":27,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401073+0000","last_change":"2024-09-13T14:12:31.085929+0000","last_active":"2024-09-13T14:14:23.401073+0000","last_peered":"2024-09-13T14:14:23.401073+0000","last_clean":"2024-09-13T14:14:23.401073+0000","last_became_active":"2024-09-13T14:12:26.296739+0000","last_became_peered":"2024-09-13T14:12:26.296739+0000","last_unstale":"2024-09-13T14:14:23.401073+0000","last_undegraded":"2024-09-13T14:14:23.401073+0000","last_fullsized":"2024-09-13T14:14:23.401073+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:14.744825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042041700000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397871+0000","last_change":"2024-09-13T14:13:41.131216+0000","last_active":"2024-09-13T14:14:23.397871+0000","last_peered":"2024-09-13T14:14:23.397871+0000","last_clean":"2024-09-13T14:14:23.397871+0000","last_became_active":"2024-09-13T14:13:41.130942+0000","last_became_peered":"2024-09-13T14:13:41.130942+0000","last_unstale":"2024-09-13T14:14:23.397871+0000","last_undegraded":"2024-09-13T14:14:23.397871+0000","last_fullsized":"2024-09-13T14:14:23.397871+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.309907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401035+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:14:23.401035+0000","last_peered":"2024-09-13T14:14:23.401035+0000","last_clean":"2024-09-13T14:14:23.401035+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:14:23.401035+0000","last_undegraded":"2024-09-13T14:14:23.401035+0000","last_fullsized":"2024-09-13T14:14:23.401035+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433692+0000","last_change":"2024-09-13T14:12:31.171913+0000","last_active":"2024-09-13T14:14:23.433692+0000","last_peered":"2024-09-13T14:14:23.433692+0000","last_clean":"2024-09-13T14:14:23.433692+0000","last_became_active":"2024-09-13T14:12:26.551516+0000","last_became_peered":"2024-09-13T14:12:26.551516+0000","last_unstale":"2024-09-13T14:14:23.433692+0000","last_undegraded":"2024-09-13T14:14:23.433692+0000","last_fullsized":"2024-09-13T14:14:23.433692+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.648647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029229000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403317+0000","last_change":"2024-09-13T14:13:41.132884+0000","last_active":"2024-09-13T14:14:23.403317+0000","last_peered":"2024-09-13T14:14:23.403317+0000","last_clean":"2024-09-13T14:14:23.403317+0000","last_became_active":"2024-09-13T14:13:41.132557+0000","last_became_peered":"2024-09-13T14:13:41.132557+0000","last_unstale":"2024-09-13T14:14:23.403317+0000","last_undegraded":"2024-09-13T14:14:23.403317+0000","last_fullsized":"2024-09-13T14:14:23.403317+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:01.648275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403621+0000","last_change":"2024-09-13T14:13:42.150182+0000","last_active":"2024-09-13T14:14:23.403621+0000","last_peered":"2024-09-13T14:14:23.403621+0000","last_clean":"2024-09-13T14:14:23.403621+0000","last_became_active":"2024-09-13T14:13:42.149891+0000","last_became_peered":"2024-09-13T14:13:42.149891+0000","last_unstale":"2024-09-13T14:14:23.403621+0000","last_undegraded":"2024-09-13T14:14:23.403621+0000","last_fullsized":"2024-09-13T14:14:23.403621+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:28.793763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398831+0000","last_change":"2024-09-13T14:12:31.081764+0000","last_active":"2024-09-13T14:14:23.398831+0000","last_peered":"2024-09-13T14:14:23.398831+0000","last_clean":"2024-09-13T14:14:23.398831+0000","last_became_active":"2024-09-13T14:12:26.551472+0000","last_became_peered":"2024-09-13T14:12:26.551472+0000","last_unstale":"2024-09-13T14:14:23.398831+0000","last_undegraded":"2024-09-13T14:14:23.398831+0000","last_fullsized":"2024-09-13T14:14:23.398831+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:04:11.250981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039453399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400649+0000","last_change":"2024-09-13T14:13:41.127892+0000","last_active":"2024-09-13T14:14:23.400649+0000","last_peered":"2024-09-13T14:14:23.400649+0000","last_clean":"2024-09-13T14:14:23.400649+0000","last_became_active":"2024-09-13T14:13:41.127621+0000","last_became_peered":"2024-09-13T14:13:41.127621+0000","last_unstale":"2024-09-13T14:14:23.400649+0000","last_undegraded":"2024-09-13T14:14:23.400649+0000","last_fullsized":"2024-09-13T14:14:23.400649+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:56:53.454219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.419344+0000","last_change":"2024-09-13T14:14:24.419344+0000","last_active":"2024-09-13T14:14:24.419344+0000","last_peered":"2024-09-13T14:14:24.419344+0000","last_clean":"2024-09-13T14:14:24.419344+0000","last_became_active":"2024-09-13T14:14:24.418981+0000","last_became_peered":"2024-09-13T14:14:24.418981+0000","last_unstale":"2024-09-13T14:14:24.419344+0000","last_undegraded":"2024-09-13T14:14:24.419344+0000","last_fullsized":"2024-09-13T14:14:24.419344+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:36.496394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":28,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401133+0000","last_change":"2024-09-13T14:12:31.086545+0000","last_active":"2024-09-13T14:14:23.401133+0000","last_peered":"2024-09-13T14:14:23.401133+0000","last_clean":"2024-09-13T14:14:23.401133+0000","last_became_active":"2024-09-13T14:12:26.297541+0000","last_became_peered":"2024-09-13T14:12:26.297541+0000","last_unstale":"2024-09-13T14:14:23.401133+0000","last_undegraded":"2024-09-13T14:14:23.401133+0000","last_fullsized":"2024-09-13T14:14:23.401133+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:54.129876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001036726,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398780+0000","last_change":"2024-09-13T14:13:41.126747+0000","last_active":"2024-09-13T14:14:23.398780+0000","last_peered":"2024-09-13T14:14:23.398780+0000","last_clean":"2024-09-13T14:14:23.398780+0000","last_became_active":"2024-09-13T14:13:41.126465+0000","last_became_peered":"2024-09-13T14:13:41.126465+0000","last_unstale":"2024-09-13T14:14:23.398780+0000","last_undegraded":"2024-09-13T14:14:23.398780+0000","last_fullsized":"2024-09-13T14:14:23.398780+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:57.216756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400984+0000","last_change":"2024-09-13T14:13:41.150310+0000","last_active":"2024-09-13T14:14:23.400984+0000","last_peered":"2024-09-13T14:14:23.400984+0000","last_clean":"2024-09-13T14:14:23.400984+0000","last_became_active":"2024-09-13T14:13:41.150041+0000","last_became_peered":"2024-09-13T14:13:41.150041+0000","last_unstale":"2024-09-13T14:14:23.400984+0000","last_undegraded":"2024-09-13T14:14:23.400984+0000","last_fullsized":"2024-09-13T14:14:23.400984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:29.604979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401829+0000","last_change":"2024-09-13T14:13:41.139508+0000","last_active":"2024-09-13T14:14:23.401829+0000","last_peered":"2024-09-13T14:14:23.401829+0000","last_clean":"2024-09-13T14:14:23.401829+0000","last_became_active":"2024-09-13T14:13:41.139227+0000","last_became_peered":"2024-09-13T14:13:41.139227+0000","last_unstale":"2024-09-13T14:14:23.401829+0000","last_undegraded":"2024-09-13T14:14:23.401829+0000","last_fullsized":"2024-09-13T14:14:23.401829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:34.799511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401102+0000","last_change":"2024-09-13T14:12:31.081654+0000","last_active":"2024-09-13T14:14:23.401102+0000","last_peered":"2024-09-13T14:14:23.401102+0000","last_clean":"2024-09-13T14:14:23.401102+0000","last_became_active":"2024-09-13T14:12:26.551473+0000","last_became_peered":"2024-09-13T14:12:26.551473+0000","last_unstale":"2024-09-13T14:14:23.401102+0000","last_undegraded":"2024-09-13T14:14:23.401102+0000","last_fullsized":"2024-09-13T14:14:23.401102+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:42:25.018815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000346795,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400654+0000","last_change":"2024-09-13T14:13:41.149515+0000","last_active":"2024-09-13T14:14:23.400654+0000","last_peered":"2024-09-13T14:14:23.400654+0000","last_clean":"2024-09-13T14:14:23.400654+0000","last_became_active":"2024-09-13T14:13:41.149103+0000","last_became_peered":"2024-09-13T14:13:41.149103+0000","last_unstale":"2024-09-13T14:14:23.400654+0000","last_undegraded":"2024-09-13T14:14:23.400654+0000","last_fullsized":"2024-09-13T14:14:23.400654+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:51.077174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401049+0000","last_change":"2024-09-13T14:13:41.129848+0000","last_active":"2024-09-13T14:14:23.401049+0000","last_peered":"2024-09-13T14:14:23.401049+0000","last_clean":"2024-09-13T14:14:23.401049+0000","last_became_active":"2024-09-13T14:13:41.129548+0000","last_became_peered":"2024-09-13T14:13:41.129548+0000","last_unstale":"2024-09-13T14:14:23.401049+0000","last_undegraded":"2024-09-13T14:14:23.401049+0000","last_fullsized":"2024-09-13T14:14:23.401049+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:40:26.100813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433967+0000","last_change":"2024-09-13T14:12:31.172062+0000","last_active":"2024-09-13T14:14:23.433967+0000","last_peered":"2024-09-13T14:14:23.433967+0000","last_clean":"2024-09-13T14:14:23.433967+0000","last_became_active":"2024-09-13T14:12:26.816843+0000","last_became_peered":"2024-09-13T14:12:26.816843+0000","last_unstale":"2024-09-13T14:14:23.433967+0000","last_undegraded":"2024-09-13T14:14:23.433967+0000","last_fullsized":"2024-09-13T14:14:23.433967+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:53.145885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000343452,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403533+0000","last_change":"2024-09-13T14:13:42.150968+0000","last_active":"2024-09-13T14:14:23.403533+0000","last_peered":"2024-09-13T14:14:23.403533+0000","last_clean":"2024-09-13T14:14:23.403533+0000","last_became_active":"2024-09-13T14:13:42.150702+0000","last_became_peered":"2024-09-13T14:13:42.150702+0000","last_unstale":"2024-09-13T14:14:23.403533+0000","last_undegraded":"2024-09-13T14:14:23.403533+0000","last_fullsized":"2024-09-13T14:14:23.403533+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:48.691675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398376+0000","last_change":"2024-09-13T14:13:41.146075+0000","last_active":"2024-09-13T14:14:23.398376+0000","last_peered":"2024-09-13T14:14:23.398376+0000","last_clean":"2024-09-13T14:14:23.398376+0000","last_became_active":"2024-09-13T14:13:41.145706+0000","last_became_peered":"2024-09-13T14:13:41.145706+0000","last_unstale":"2024-09-13T14:14:23.398376+0000","last_undegraded":"2024-09-13T14:14:23.398376+0000","last_fullsized":"2024-09-13T14:14:23.398376+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:25.189062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401294+0000","last_change":"2024-09-13T14:12:31.081944+0000","last_active":"2024-09-13T14:14:23.401294+0000","last_peered":"2024-09-13T14:14:23.401294+0000","last_clean":"2024-09-13T14:14:23.401294+0000","last_became_active":"2024-09-13T14:12:26.816862+0000","last_became_peered":"2024-09-13T14:12:26.816862+0000","last_unstale":"2024-09-13T14:14:23.401294+0000","last_undegraded":"2024-09-13T14:14:23.401294+0000","last_fullsized":"2024-09-13T14:14:23.401294+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:43.703026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062990400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397651+0000","last_change":"2024-09-13T14:13:42.152844+0000","last_active":"2024-09-13T14:14:23.397651+0000","last_peered":"2024-09-13T14:14:23.397651+0000","last_clean":"2024-09-13T14:14:23.397651+0000","last_became_active":"2024-09-13T14:13:42.152016+0000","last_became_peered":"2024-09-13T14:13:42.152016+0000","last_unstale":"2024-09-13T14:14:23.397651+0000","last_undegraded":"2024-09-13T14:14:23.397651+0000","last_fullsized":"2024-09-13T14:14:23.397651+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:58.105884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403723+0000","last_change":"2024-09-13T14:13:41.126928+0000","last_active":"2024-09-13T14:14:23.403723+0000","last_peered":"2024-09-13T14:14:23.403723+0000","last_clean":"2024-09-13T14:14:23.403723+0000","last_became_active":"2024-09-13T14:13:41.126243+0000","last_became_peered":"2024-09-13T14:13:41.126243+0000","last_unstale":"2024-09-13T14:14:23.403723+0000","last_undegraded":"2024-09-13T14:14:23.403723+0000","last_fullsized":"2024-09-13T14:14:23.403723+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:48.937352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401570+0000","last_change":"2024-09-13T14:12:31.087292+0000","last_active":"2024-09-13T14:14:23.401570+0000","last_peered":"2024-09-13T14:14:23.401570+0000","last_clean":"2024-09-13T14:14:23.401570+0000","last_became_active":"2024-09-13T14:12:26.296590+0000","last_became_peered":"2024-09-13T14:12:26.296590+0000","last_unstale":"2024-09-13T14:14:23.401570+0000","last_undegraded":"2024-09-13T14:14:23.401570+0000","last_fullsized":"2024-09-13T14:14:23.401570+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:02.313429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026736199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398216+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:14:23.398216+0000","last_peered":"2024-09-13T14:14:23.398216+0000","last_clean":"2024-09-13T14:14:23.398216+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:14:23.398216+0000","last_undegraded":"2024-09-13T14:14:23.398216+0000","last_fullsized":"2024-09-13T14:14:23.398216+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398698+0000","last_change":"2024-09-13T14:13:41.150310+0000","last_active":"2024-09-13T14:14:23.398698+0000","last_peered":"2024-09-13T14:14:23.398698+0000","last_clean":"2024-09-13T14:14:23.398698+0000","last_became_active":"2024-09-13T14:13:41.150035+0000","last_became_peered":"2024-09-13T14:13:41.150035+0000","last_unstale":"2024-09-13T14:14:23.398698+0000","last_undegraded":"2024-09-13T14:14:23.398698+0000","last_fullsized":"2024-09-13T14:14:23.398698+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:26.164328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"64'6","reported_seq":20,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398490+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:14:23.398490+0000","last_peered":"2024-09-13T14:14:23.398490+0000","last_clean":"2024-09-13T14:14:23.398490+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:14:23.398490+0000","last_undegraded":"2024-09-13T14:14:23.398490+0000","last_fullsized":"2024-09-13T14:14:23.398490+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400642+0000","last_change":"2024-09-13T14:13:41.147747+0000","last_active":"2024-09-13T14:14:23.400642+0000","last_peered":"2024-09-13T14:14:23.400642+0000","last_clean":"2024-09-13T14:14:23.400642+0000","last_became_active":"2024-09-13T14:13:41.147209+0000","last_became_peered":"2024-09-13T14:13:41.147209+0000","last_unstale":"2024-09-13T14:14:23.400642+0000","last_undegraded":"2024-09-13T14:14:23.400642+0000","last_fullsized":"2024-09-13T14:14:23.400642+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:55.514702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":17,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.433864+0000","last_change":"2024-09-13T14:13:41.146093+0000","last_active":"2024-09-13T14:14:23.433864+0000","last_peered":"2024-09-13T14:14:23.433864+0000","last_clean":"2024-09-13T14:14:23.433864+0000","last_became_active":"2024-09-13T14:13:41.145816+0000","last_became_peered":"2024-09-13T14:13:41.145816+0000","last_unstale":"2024-09-13T14:14:23.433864+0000","last_undegraded":"2024-09-13T14:14:23.433864+0000","last_fullsized":"2024-09-13T14:14:23.433864+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:19.263045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401772+0000","last_change":"2024-09-13T14:13:41.133453+0000","last_active":"2024-09-13T14:14:23.401772+0000","last_peered":"2024-09-13T14:14:23.401772+0000","last_clean":"2024-09-13T14:14:23.401772+0000","last_became_active":"2024-09-13T14:13:41.133209+0000","last_became_peered":"2024-09-13T14:13:41.133209+0000","last_unstale":"2024-09-13T14:14:23.401772+0000","last_undegraded":"2024-09-13T14:14:23.401772+0000","last_fullsized":"2024-09-13T14:14:23.401772+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:33.498683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397649+0000","last_change":"2024-09-13T14:13:42.489086+0000","last_active":"2024-09-13T14:14:23.397649+0000","last_peered":"2024-09-13T14:14:23.397649+0000","last_clean":"2024-09-13T14:14:23.397649+0000","last_became_active":"2024-09-13T14:13:42.488762+0000","last_became_peered":"2024-09-13T14:13:42.488762+0000","last_unstale":"2024-09-13T14:14:23.397649+0000","last_undegraded":"2024-09-13T14:14:23.397649+0000","last_fullsized":"2024-09-13T14:14:23.397649+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:59:18.584820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403921+0000","last_change":"2024-09-13T14:13:41.136346+0000","last_active":"2024-09-13T14:14:23.403921+0000","last_peered":"2024-09-13T14:14:23.403921+0000","last_clean":"2024-09-13T14:14:23.403921+0000","last_became_active":"2024-09-13T14:13:41.135422+0000","last_became_peered":"2024-09-13T14:13:41.135422+0000","last_unstale":"2024-09-13T14:14:23.403921+0000","last_undegraded":"2024-09-13T14:14:23.403921+0000","last_fullsized":"2024-09-13T14:14:23.403921+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:11.340917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397625+0000","last_change":"2024-09-13T14:13:42.489075+0000","last_active":"2024-09-13T14:14:23.397625+0000","last_peered":"2024-09-13T14:14:23.397625+0000","last_clean":"2024-09-13T14:14:23.397625+0000","last_became_active":"2024-09-13T14:13:42.488738+0000","last_became_peered":"2024-09-13T14:13:42.488738+0000","last_unstale":"2024-09-13T14:14:23.397625+0000","last_undegraded":"2024-09-13T14:14:23.397625+0000","last_fullsized":"2024-09-13T14:14:23.397625+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:35:11.076752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401489+0000","last_change":"2024-09-13T14:13:41.150482+0000","last_active":"2024-09-13T14:14:23.401489+0000","last_peered":"2024-09-13T14:14:23.401489+0000","last_clean":"2024-09-13T14:14:23.401489+0000","last_became_active":"2024-09-13T14:13:41.150196+0000","last_became_peered":"2024-09-13T14:13:41.150196+0000","last_unstale":"2024-09-13T14:14:23.401489+0000","last_undegraded":"2024-09-13T14:14:23.401489+0000","last_fullsized":"2024-09-13T14:14:23.401489+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:31.060006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397924+0000","last_change":"2024-09-13T14:13:41.141497+0000","last_active":"2024-09-13T14:14:23.397924+0000","last_peered":"2024-09-13T14:14:23.397924+0000","last_clean":"2024-09-13T14:14:23.397924+0000","last_became_active":"2024-09-13T14:13:41.139267+0000","last_became_peered":"2024-09-13T14:13:41.139267+0000","last_unstale":"2024-09-13T14:14:23.397924+0000","last_undegraded":"2024-09-13T14:14:23.397924+0000","last_fullsized":"2024-09-13T14:14:23.397924+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:40.482904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403614+0000","last_change":"2024-09-13T14:13:42.150796+0000","last_active":"2024-09-13T14:14:23.403614+0000","last_peered":"2024-09-13T14:14:23.403614+0000","last_clean":"2024-09-13T14:14:23.403614+0000","last_became_active":"2024-09-13T14:13:42.150512+0000","last_became_peered":"2024-09-13T14:13:42.150512+0000","last_unstale":"2024-09-13T14:14:23.403614+0000","last_undegraded":"2024-09-13T14:14:23.403614+0000","last_fullsized":"2024-09-13T14:14:23.403614+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:11.912670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.400500+0000","last_change":"2024-09-13T14:13:41.146409+0000","last_active":"2024-09-13T14:14:23.400500+0000","last_peered":"2024-09-13T14:14:23.400500+0000","last_clean":"2024-09-13T14:14:23.400500+0000","last_became_active":"2024-09-13T14:13:41.146125+0000","last_became_peered":"2024-09-13T14:13:41.146125+0000","last_unstale":"2024-09-13T14:14:23.400500+0000","last_undegraded":"2024-09-13T14:14:23.400500+0000","last_fullsized":"2024-09-13T14:14:23.400500+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:28.588056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398171+0000","last_change":"2024-09-13T14:13:42.607992+0000","last_active":"2024-09-13T14:14:23.398171+0000","last_peered":"2024-09-13T14:14:23.398171+0000","last_clean":"2024-09-13T14:14:23.398171+0000","last_became_active":"2024-09-13T14:13:42.607575+0000","last_became_peered":"2024-09-13T14:13:42.607575+0000","last_unstale":"2024-09-13T14:14:23.398171+0000","last_undegraded":"2024-09-13T14:14:23.398171+0000","last_fullsized":"2024-09-13T14:14:23.398171+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:53.106609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398430+0000","last_change":"2024-09-13T14:13:41.130505+0000","last_active":"2024-09-13T14:14:23.398430+0000","last_peered":"2024-09-13T14:14:23.398430+0000","last_clean":"2024-09-13T14:14:23.398430+0000","last_became_active":"2024-09-13T14:13:41.130120+0000","last_became_peered":"2024-09-13T14:13:41.130120+0000","last_unstale":"2024-09-13T14:14:23.398430+0000","last_undegraded":"2024-09-13T14:14:23.398430+0000","last_fullsized":"2024-09-13T14:14:23.398430+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:50.637901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398825+0000","last_change":"2024-09-13T14:13:41.147138+0000","last_active":"2024-09-13T14:14:23.398825+0000","last_peered":"2024-09-13T14:14:23.398825+0000","last_clean":"2024-09-13T14:14:23.398825+0000","last_became_active":"2024-09-13T14:13:41.146828+0000","last_became_peered":"2024-09-13T14:13:41.146828+0000","last_unstale":"2024-09-13T14:14:23.398825+0000","last_undegraded":"2024-09-13T14:14:23.398825+0000","last_fullsized":"2024-09-13T14:14:23.398825+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:26.095253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403628+0000","last_change":"2024-09-13T14:13:41.130035+0000","last_active":"2024-09-13T14:14:23.403628+0000","last_peered":"2024-09-13T14:14:23.403628+0000","last_clean":"2024-09-13T14:14:23.403628+0000","last_became_active":"2024-09-13T14:13:41.129732+0000","last_became_peered":"2024-09-13T14:13:41.129732+0000","last_unstale":"2024-09-13T14:14:23.403628+0000","last_undegraded":"2024-09-13T14:14:23.403628+0000","last_fullsized":"2024-09-13T14:14:23.403628+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:30.425166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398640+0000","last_change":"2024-09-13T14:13:41.150436+0000","last_active":"2024-09-13T14:14:23.398640+0000","last_peered":"2024-09-13T14:14:23.398640+0000","last_clean":"2024-09-13T14:14:23.398640+0000","last_became_active":"2024-09-13T14:13:41.150174+0000","last_became_peered":"2024-09-13T14:13:41.150174+0000","last_unstale":"2024-09-13T14:14:23.398640+0000","last_undegraded":"2024-09-13T14:14:23.398640+0000","last_fullsized":"2024-09-13T14:14:23.398640+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:16.384357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401309+0000","last_change":"2024-09-13T14:13:41.131340+0000","last_active":"2024-09-13T14:14:23.401309+0000","last_peered":"2024-09-13T14:14:23.401309+0000","last_clean":"2024-09-13T14:14:23.401309+0000","last_became_active":"2024-09-13T14:13:41.130996+0000","last_became_peered":"2024-09-13T14:13:41.130996+0000","last_unstale":"2024-09-13T14:14:23.401309+0000","last_undegraded":"2024-09-13T14:14:23.401309+0000","last_fullsized":"2024-09-13T14:14:23.401309+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:09.294099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398339+0000","last_change":"2024-09-13T14:13:41.149509+0000","last_active":"2024-09-13T14:14:23.398339+0000","last_peered":"2024-09-13T14:14:23.398339+0000","last_clean":"2024-09-13T14:14:23.398339+0000","last_became_active":"2024-09-13T14:13:41.149217+0000","last_became_peered":"2024-09-13T14:13:41.149217+0000","last_unstale":"2024-09-13T14:14:23.398339+0000","last_undegraded":"2024-09-13T14:14:23.398339+0000","last_fullsized":"2024-09-13T14:14:23.398339+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:59.273576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":16,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.401741+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:14:23.401741+0000","last_peered":"2024-09-13T14:14:23.401741+0000","last_clean":"2024-09-13T14:14:23.401741+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:14:23.401741+0000","last_undegraded":"2024-09-13T14:14:23.401741+0000","last_fullsized":"2024-09-13T14:14:23.401741+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.397829+0000","last_change":"2024-09-13T14:13:42.151860+0000","last_active":"2024-09-13T14:14:23.397829+0000","last_peered":"2024-09-13T14:14:23.397829+0000","last_clean":"2024-09-13T14:14:23.397829+0000","last_became_active":"2024-09-13T14:13:42.151429+0000","last_became_peered":"2024-09-13T14:13:42.151429+0000","last_unstale":"2024-09-13T14:14:23.397829+0000","last_undegraded":"2024-09-13T14:14:23.397829+0000","last_fullsized":"2024-09-13T14:14:23.397829+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:24:41.330481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":15,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.398351+0000","last_change":"2024-09-13T14:13:41.128835+0000","last_active":"2024-09-13T14:14:23.398351+0000","last_peered":"2024-09-13T14:14:23.398351+0000","last_clean":"2024-09-13T14:14:23.398351+0000","last_became_active":"2024-09-13T14:13:41.128440+0000","last_became_peered":"2024-09-13T14:13:41.128440+0000","last_unstale":"2024-09-13T14:14:23.398351+0000","last_undegraded":"2024-09-13T14:14:23.398351+0000","last_fullsized":"2024-09-13T14:14:23.398351+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:06.588300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:14:24.758890+0000","last_change":"2024-09-13T14:14:24.758890+0000","last_active":"2024-09-13T14:14:24.758890+0000","last_peered":"2024-09-13T14:14:24.758890+0000","last_clean":"2024-09-13T14:14:24.758890+0000","last_became_active":"2024-09-13T14:14:24.758588+0000","last_became_peered":"2024-09-13T14:14:24.758588+0000","last_unstale":"2024-09-13T14:14:24.758890+0000","last_undegraded":"2024-09-13T14:14:24.758890+0000","last_fullsized":"2024-09-13T14:14:24.758890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:25.781156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":18,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:14:23.403903+0000","last_change":"2024-09-13T14:13:41.136986+0000","last_active":"2024-09-13T14:14:23.403903+0000","last_peered":"2024-09-13T14:14:23.403903+0000","last_clean":"2024-09-13T14:14:23.403903+0000","last_became_active":"2024-09-13T14:13:41.136609+0000","last_became_peered":"2024-09-13T14:13:41.136609+0000","last_unstale":"2024-09-13T14:14:23.403903+0000","last_undegraded":"2024-09-13T14:14:23.403903+0000","last_fullsized":"2024-09-13T14:14:23.403903+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:46.403461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":888643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.751,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.53600000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78300000000000003,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68700000000000006,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.57199999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66700000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58399999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.68899999999999995,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.503},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71599999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.48099999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.41099999999999998,"15min":0.40500000000000003},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.374}]},{"osd":5,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.432,"15min":0.432},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.46100000000000002,"15min":0.45700000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.39800000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.54500000000000004}]}]},{"osd":6,"up_from":43,"seq":184683593769,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298604,"kb_used_data":1248,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446548,"statfs":{"total":95995035648,"available":95689265152,"internally_reserved":0,"allocated":1277952,"data_stored":887957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.504,"15min":0.49399999999999999},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51800000000000002,"15min":0.502},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72099999999999997,"5min":1.083,"15min":1.083},"last":0.64500000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.497,"15min":0.48799999999999999},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72099999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52200000000000002,"15min":0.51600000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54900000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.42299999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53000000000000003,"15min":0.52200000000000002},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56999999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52000000000000002,"15min":0.51400000000000001},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66900000000000004,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76100000000000001,"5min":1.117,"15min":1.117},"last":0.48399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.41199999999999998,"15min":0.39400000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.499}]},{"osd":5,"last update":"Fri Sep 13 14:14:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.441,"15min":0.42099999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70899999999999996,"5min":0.754,"15min":0.754},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.45300000000000001,"15min":0.432},"min":{"1min":0.30599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68999999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46800000000000003}]},{"osd":7,"last update":"Fri Sep 13 14:13:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.503,"15min":0.495},"min":{"1min":0.36699999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.58099999999999996}]}]},{"osd":5,"up_from":37,"seq":158913789999,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":675840,"data_stored":296536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.56499999999999995},"min":{"1min":0.42399999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.66800000000000004,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54700000000000004,"15min":0.56499999999999995},"min":{"1min":0.40999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72099999999999997,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.40300000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73199999999999998,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.44800000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72699999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.44800000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.55200000000000005,"15min":0.56399999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70499999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.56000000000000005,"15min":0.57299999999999995},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73999999999999999,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56699999999999995,"15min":0.58199999999999996},"min":{"1min":0.40699999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.66400000000000003,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.47899999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.65700000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59499999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:14:11 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.41399999999999998,"15min":0.39600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.41599999999999998,"15min":0.39600000000000002},"min":{"1min":0.36899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.432,"15min":0.42499999999999999},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.43099999999999999,"15min":0.41899999999999998},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68600000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.64800000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47599999999999998,"15min":0.46600000000000003},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.47899999999999998,"15min":0.47199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.58099999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":298205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.51300000000000001,"15min":0.52700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64000000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.375},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.63500000000000001,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.39800000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.50800000000000001,"15min":0.51800000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65300000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.505,"15min":0.51700000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68600000000000005,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.48299999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.50800000000000001,"15min":0.51100000000000001},"min":{"1min":0.371,"5min":0.311,"15min":0.311},"max":{"1min":0.69799999999999995,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51000000000000001,"15min":0.51400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68899999999999995,"5min":0.879,"15min":0.879},"last":0.46200000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52200000000000002,"15min":0.52900000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66800000000000004,"5min":0.753,"15min":0.753},"last":0.496},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54300000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:14:10 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.36899999999999999,"15min":0.34200000000000003},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.375,"15min":0.34899999999999998},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63200000000000001,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.53200000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.40799999999999997,"15min":0.39300000000000002},"min":{"1min":0.26100000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66000000000000003,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.46700000000000003,"15min":0.45500000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.71599999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.46899999999999997,"15min":0.46500000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.755,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.497,"5min":0.48099999999999998,"15min":0.47799999999999998},"min":{"1min":0.30199999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.58099999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":675840,"data_stored":296597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.376,"15min":0.377},"min":{"1min":0.21299999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40400000000000003,"15min":0.40200000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.45700000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.25,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.372},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.28799999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45400000000000001,"15min":0.46100000000000002},"min":{"1min":0.26800000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.51800000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:13:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83099999999999996,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:14:08 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.623,"15min":0.61899999999999999},"min":{"1min":0.45800000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.61299999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:14:35 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.42899999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.63100000000000001,"15min":0.626},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59999999999999998,"15min":0.59399999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59899999999999998},"min":{"1min":0.49399999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.53900000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298016,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447136,"statfs":{"total":95995035648,"available":95689867264,"internally_reserved":0,"allocated":679936,"data_stored":297111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.36799999999999999,"15min":0.36399999999999999},"min":{"1min":0.247,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.52600000000000002,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.379,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.26200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.374,"15min":0.374},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.248},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.58399999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.29499999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:14:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.439,"15min":0.45600000000000002},"min":{"1min":0.30599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.315},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.42499999999999999,"15min":0.44500000000000001},"min":{"1min":0.29299999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:13:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.60399999999999998,"15min":0.59399999999999997},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78500000000000003,"5min":1.024,"15min":1.024},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71499999999999997,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.65500000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80900000000000005,"5min":1.101,"15min":1.101},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77200000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:14:35 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61399999999999999,"15min":0.63700000000000001},"min":{"1min":0.39300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80900000000000005,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.622,"15min":0.64500000000000002},"min":{"1min":0.45100000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.85199999999999998,"5min":1.04,"15min":1.04},"last":0.60299999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.625,"15min":0.63800000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.746,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60599999999999998,"15min":0.61499999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77500000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.63200000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":675840,"data_stored":296526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.373,"15min":0.35299999999999998},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.47399999999999998,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.379,"5min":0.376,"15min":0.35799999999999998},"min":{"1min":0.30499999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.46000000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.33300000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.38200000000000001,"15min":0.38500000000000001},"min":{"1min":0.25700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.54500000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38700000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:14:28 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.42199999999999999,"15min":0.435},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.51000000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.37},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.435,"15min":0.44900000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59699999999999998,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.30399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:13:42 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.61899999999999999,"15min":0.61399999999999999},"min":{"1min":0.45800000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.47699999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.53500000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.63600000000000001,"15min":0.64100000000000001},"min":{"1min":0.433,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0069999999999999,"5min":1.129,"15min":1.129},"last":0.51000000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:14:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.42899999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76700000000000002,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63800000000000001,"15min":0.64600000000000002},"min":{"1min":0.51100000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81299999999999994,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.48099999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:13:57 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.90500000000000003,"5min":1.252,"15min":1.252},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.97099999999999997,"5min":1.415,"15min":1.415},"last":0.497}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298600,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446552,"statfs":{"total":95995035648,"available":95689269248,"internally_reserved":0,"allocated":1269760,"data_stored":887152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:13:44 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.34499999999999997},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.374,"15min":0.35399999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:14:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.25600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.42399999999999999,"15min":0.40799999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.66100000000000003,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.28699999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:14:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.371},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.34100000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:13:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60599999999999998,"15min":0.59799999999999998},"min":{"1min":0.39900000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.52000000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:14:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.436,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.84399999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.629,"15min":0.626},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.61899999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:14:36 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.61199999999999999,"15min":0.60799999999999998},"min":{"1min":0.47899999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.629,"15min":0.624},"min":{"1min":0.47099999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.68600000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.61299999999999999,"15min":0.60599999999999998},"min":{"1min":0.40300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.63200000000000001,"15min":0.625},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:14:44.103 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:14:44.103 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:14:44.103 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:14:44.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:14:44.362 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:44 smithi017 ceph-mon[29399]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:44 smithi017 ceph-mon[29399]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:44 smithi017 ceph-mon[38255]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:44 smithi017 ceph-mon[38255]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:44 smithi189 ceph-mon[33294]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:44 smithi189 ceph-mon[33294]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:14:46.264 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:14:46.265 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:14:46.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:46 smithi017 ceph-mon[29399]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:46.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:46 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/715447338' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:14:46.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:46 smithi017 ceph-mon[38255]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:46.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:46 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/715447338' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:14:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:46 smithi189 ceph-mon[33294]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:46 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/715447338' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:14:46.957 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:14:46.957 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:14:46.970 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-13T14:14:46.970 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:14:46.979 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:14:46.979 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:14:46.980 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T14:14:46.985 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T14:14:46.985 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:14:46.985 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:14:46.986 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:14:46.986 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T14:14:46.986 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2024-09-13T14:14:46.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-13T14:14:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:14:47.005 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:14:47.005 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:14:47.061 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:14:47.061 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:14:47.118 INFO:tasks.workunit:timeout=3h 2024-09-13T14:14:47.204 INFO:tasks.workunit:cleanup=True 2024-09-13T14:14:47.205 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:14:47.222 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:14:47.251 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:47.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:47] "GET /metrics HTTP/1.1" 200 239783 "" "Prometheus/2.43.0" 2024-09-13T14:14:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:48 smithi189 ceph-mon[33294]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:48.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:48 smithi017 ceph-mon[29399]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:48.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:48 smithi017 ceph-mon[38255]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:49.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE Bus STOPPING 2024-09-13T14:14:49.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:14:49.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE Bus STOPPED 2024-09-13T14:14:49.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE Bus STARTING 2024-09-13T14:14:49.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE Serving on http://:::9283 2024-09-13T14:14:49.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:14:49] ENGINE Bus STARTED 2024-09-13T14:14:49.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T14:14:50.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:14:50.305 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:50.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3596368063' entity='client.admin' 2024-09-13T14:14:50.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3596368063' entity='client.admin' 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:50 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3596368063' entity='client.admin' 2024-09-13T14:14:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:50 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:52.335 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:14:52.408 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:14:52.410 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:14:52.411 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:14:52.477 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:52 smithi017 ceph-mon[29399]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:52.477 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:52 smithi017 ceph-mon[38255]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:52 smithi189 ceph-mon[33294]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:53 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:53 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:53 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:14:53.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T14:14:53.993 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:14:54.040 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:54.083 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:14:54.124 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:14:54.126 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:14:54.126 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:14:54.166 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:14:54.169 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:14:54.169 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:14:54.185 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-09-13T14:14:54.187 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-09-13T14:14:54.187 DEBUG:teuthology.orchestra.run.smithi017:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-09-13T14:14:54.263 INFO:tasks.workunit.client.0.smithi017.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-09-13T14:14:54.263 INFO:tasks.workunit.client.0.smithi017.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-09-13T14:14:54.263 INFO:tasks.workunit.client.0.smithi017.stderr:+ TMPDIR=/tmp/rbd_import_export_82721 2024-09-13T14:14:54.263 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_82721 2024-09-13T14:14:54.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir /tmp/rbd_import_export_82721 2024-09-13T14:14:54.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ trap 'rm -rf /tmp/rbd_import_export_82721' INT TERM EXIT 2024-09-13T14:14:54.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir foo.82721 2024-09-13T14:14:54.265 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import foo.82721 foo.dir 2024-09-13T14:14:54.355 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: cannot import a directory 2024-09-13T14:14:54.355 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 0% complete...failed. 2024-09-13T14:14:54.355 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: import failed: (21) Is a directory 2024-09-13T14:14:54.358 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:14:54.358 INFO:tasks.workunit.client.0.smithi017.stderr:+ rmdir foo.82721 2024-09-13T14:14:54.370 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_82721/img bs=1k count=1 seek=10 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.5597e-05 s, 22.5 MB/s 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_82721/img bs=1k count=10 seek=100 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2024-09-13T14:14:54.371 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2024-09-13T14:14:54.372 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.5691e-05 s, 184 MB/s 2024-09-13T14:14:54.372 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_82721/img bs=1k count=100 seek=1000 2024-09-13T14:14:54.372 INFO:tasks.workunit.client.0.smithi017.stderr:60+1 records in 2024-09-13T14:14:54.372 INFO:tasks.workunit.client.0.smithi017.stderr:60+1 records out 2024-09-13T14:14:54.373 INFO:tasks.workunit.client.0.smithi017.stderr:61640 bytes (62 kB, 60 KiB) copied, 0.000185083 s, 333 MB/s 2024-09-13T14:14:54.373 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_82721/img bs=1k seek=10000 2024-09-13T14:14:54.374 INFO:tasks.workunit.client.0.smithi017.stderr:137+1 records in 2024-09-13T14:14:54.374 INFO:tasks.workunit.client.0.smithi017.stderr:137+1 records out 2024-09-13T14:14:54.374 INFO:tasks.workunit.client.0.smithi017.stderr:140928 bytes (141 kB, 138 KiB) copied, 0.000411405 s, 343 MB/s 2024-09-13T14:14:54.374 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_82721/img bs=1k seek=100000 2024-09-13T14:14:54.374 INFO:tasks.workunit.client.0.smithi017.stderr:60+1 records in 2024-09-13T14:14:54.375 INFO:tasks.workunit.client.0.smithi017.stderr:60+1 records out 2024-09-13T14:14:54.375 INFO:tasks.workunit.client.0.smithi017.stderr:61592 bytes (62 kB, 60 KiB) copied, 0.000174154 s, 354 MB/s 2024-09-13T14:14:54.375 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_82721/img bs=1k seek=1000000 2024-09-13T14:14:54.376 INFO:tasks.workunit.client.0.smithi017.stderr:154+1 records in 2024-09-13T14:14:54.376 INFO:tasks.workunit.client.0.smithi017.stderr:154+1 records out 2024-09-13T14:14:54.376 INFO:tasks.workunit.client.0.smithi017.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000532988 s, 297 MB/s 2024-09-13T14:14:54.376 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2024-09-13T14:14:54.416 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2024-09-13T14:14:54.416 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:14:54.420 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:14:54.421 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:14:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:54 smithi017 ceph-mon[29399]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:54 smithi017 ceph-mon[38255]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:54 smithi189 ceph-mon[33294]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:54.939 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-09-13T14:14:54.939 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:14:54.941 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_82721/img2 2024-09-13T14:14:55.235 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:14:55.238 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2024-09-13T14:14:55.919 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:14:55.922 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2024-09-13T14:14:56.014 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T14:14:56.017 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img2 2024-09-13T14:14:56.185 INFO:teuthology.orchestra.run.smithi017.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:56 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.655 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img3 2024-09-13T14:14:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:56 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:14:56.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:14:57.141 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img2 /tmp/rbd_import_export_82721/img3 2024-09-13T14:14:57.231 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:57.316 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - testimg 2024-09-13T14:14:57.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[29399]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:57.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[29399]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[29399]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[38255]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:57.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[38255]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-mon[38255]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.708 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:14:57.709 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:14:57.712 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_82721/img2 2024-09-13T14:14:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:57 smithi189 ceph-mon[33294]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:57 smithi189 ceph-mon[33294]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:57 smithi189 ceph-mon[33294]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:14:57.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:14:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:57] "GET /metrics HTTP/1.1" 200 239771 "" "Prometheus/2.43.0" 2024-09-13T14:14:57.980 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:14:57.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2024-09-13T14:14:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:14:58 smithi017 ceph-mon[29399]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:14:58 smithi017 ceph-mon[38255]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:14:58 smithi189 ceph-mon[33294]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:14:58.759 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-13T14:14:58.762 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2024-09-13T14:14:58.854 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-13T14:14:58.857 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img2 2024-09-13T14:14:59.446 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:14:59.446 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (82s) 19s ago 2m 17.8M - 0.25.0 c8568f914cd2 c12ac3c783a2 2024-09-13T14:14:59.446 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (2m) 37s ago 2m 80.5M - 9.4.7 954c08fa6188 da25cb53d36f 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (65s) 19s ago 64s 15.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (62s) 37s ago 62s 11.7M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443 running (6m) 37s ago 6m 399M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 0e0fb3e7d211 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:9283,8765 running (7m) 19s ago 7m 445M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 566606e62d48 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (7m) 19s ago 8m 45.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7f6206d3dbf8 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (6m) 37s ago 6m 36.8M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 6439c9112cda 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (6m) 19s ago 6m 37.8M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 67f2b6c9b084 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (116s) 19s ago 113s 14.0M - 1.5.0 0da6a335fe13 844957d581ca 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (110s) 37s ago 109s 7755k - 1.5.0 0da6a335fe13 b928aa7fd0a1 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (5m) 19s ago 5m 41.7M 4017M 17.2.7-1627-gbbdadfa3 a994b00deb3a 61ff807d7240 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (5m) 19s ago 5m 45.4M 4017M 17.2.7-1627-gbbdadfa3 a994b00deb3a b8f0ae12aacb 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (5m) 19s ago 5m 40.0M 4017M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2e705cedcfaf 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (4m) 19s ago 4m 40.4M 4017M 17.2.7-1627-gbbdadfa3 a994b00deb3a 81c4e9274ff0 2024-09-13T14:14:59.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (4m) 37s ago 4m 43.4M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:14:59.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (4m) 37s ago 4m 40.3M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:14:59.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (3m) 37s ago 3m 41.2M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:14:59.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (3m) 37s ago 3m 44.4M 4016M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:14:59.448 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (77s) 37s ago 2m 35.1M - 2.43.0 a07b618ecd1d acf8b16b2fc1 2024-09-13T14:14:59.584 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img3 2024-09-13T14:15:00.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:14:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:14:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:00.090 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img2 /tmp/rbd_import_export_82721/img3 2024-09-13T14:15:00.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-13T14:15:00.336 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd help export 2024-09-13T14:15:00.336 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -q export-format 2024-09-13T14:15:00.358 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_82721/img bs=1k count=10 seek=100 2024-09-13T14:15:00.359 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2024-09-13T14:15:00.359 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2024-09-13T14:15:00.359 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 9.1262e-05 s, 112 MB/s 2024-09-13T14:15:00.359 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:15:00.423 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:00.423 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:00.426 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap 2024-09-13T14:15:00.583 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:00 smithi017 ceph-mon[29399]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:00 smithi017 ceph-mon[29399]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:00 smithi017 ceph-mon[38255]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:00 smithi017 ceph-mon[38255]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:00 smithi189 ceph-mon[33294]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:00 smithi189 ceph-mon[33294]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:01.403 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:15:01.410 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd image-meta set testimg key1 value1 2024-09-13T14:15:01.462 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg 2024-09-13T14:15:01.505 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-09-13T14:15:01.506 INFO:tasks.workunit.client.0.smithi017.stderr: 2024-09-13T14:15:01.506 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2024-09-13T14:15:01.506 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2024-09-13T14:15:01.506 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:01.551 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:01.554 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_82721/img_v2 testimg_import 2024-09-13T14:15:02.387 INFO:teuthology.orchestra.run.smithi017.stdout:wait for minority of mons to upgrade 2024-09-13T14:15:02.413 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:15:02.416 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2024-09-13T14:15:02.453 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2024-09-13T14:15:02.453 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2024-09-13T14:15:02.453 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: id: 3a1dcff6916a 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.3a1dcff6916a 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.454 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.456 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: id: 3a1dcff6916a 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.3a1dcff6916a 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2024-09-13T14:15:02.494 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.495 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.495 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Fri Sep 13 14:15:01 2024 2024-09-13T14:15:02.495 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2024-09-13T14:15:02.497 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg_import 2024-09-13T14:15:02.539 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 'There is 1 metadatum on this image: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr: 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' ']' 2024-09-13T14:15:02.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import /tmp/rbd_import_export_82721/img_import 2024-09-13T14:15:02.579 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2024-09-13T14:15:02.583 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img_import 2024-09-13T14:15:02.583 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/img /tmp/rbd_import_export_82721/img_import 2024-09-13T14:15:02.584 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/img --format %b 2024-09-13T14:15:02.585 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=24 2024-09-13T14:15:02.585 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/img_import --format %b 2024-09-13T14:15:02.587 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2024-09-13T14:15:02.587 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2024-09-13T14:15:02.587 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2024-09-13T14:15:02.587 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-09-13T14:15:02.587 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_82721/img_snap 2024-09-13T14:15:02.628 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2024-09-13T14:15:02.631 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_82721/img_snap_import 2024-09-13T14:15:02.672 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2024-09-13T14:15:02.675 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/img_snap /tmp/rbd_import_export_82721/img_snap_import 2024-09-13T14:15:02.675 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/img_snap /tmp/rbd_import_export_82721/img_snap_import 2024-09-13T14:15:02.677 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/img_snap --format %b 2024-09-13T14:15:02.678 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=224 2024-09-13T14:15:02.678 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/img_snap_import --format %b 2024-09-13T14:15:02.679 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2024-09-13T14:15:02.679 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:02.679 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:02.679 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:15:02.679 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:02.680 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_import 2024-09-13T14:15:02.681 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_snap 2024-09-13T14:15:02.681 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_snap_import 2024-09-13T14:15:02.682 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg_import@snap 2024-09-13T14:15:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:02 smithi017 ceph-mon[29399]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:02 smithi017 ceph-mon[29399]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:15:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:02 smithi017 ceph-mon[38255]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:02 smithi017 ceph-mon[38255]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:15:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:02 smithi189 ceph-mon[33294]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 78 KiB/s wr, 67 op/s 2024-09-13T14:15:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:02 smithi189 ceph-mon[33294]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:15:03.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:15:03.376 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:03.409 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2024-09-13T14:15:03.417 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2024-09-13T14:15:03.509 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:03.512 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg@snap 2024-09-13T14:15:03.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:03 smithi017 ceph-mon[38255]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:15:03.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:03 smithi017 ceph-mon[29399]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:15:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:03 smithi189 ceph-mon[33294]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:15:04.414 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2024-09-13T14:15:04.421 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2024-09-13T14:15:04.513 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:04.516 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --order 20 /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:15:04.575 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:04.576 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:04.579 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:04.619 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:04.622 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_82721/img_v2 testimg_import 2024-09-13T14:15:04.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:04 smithi017 ceph-mon[38255]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 100 op/s 2024-09-13T14:15:04.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:04 smithi017 ceph-mon[38255]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:15:04.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:04 smithi017 ceph-mon[29399]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 100 op/s 2024-09-13T14:15:04.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:04 smithi017 ceph-mon[29399]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:15:04.687 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:15:04.691 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2024-09-13T14:15:04.691 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep order 2024-09-13T14:15:04.691 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $2}' 2024-09-13T14:15:04.691 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 20 2024-09-13T14:15:04.737 INFO:tasks.workunit.client.0.smithi017.stdout:20 2024-09-13T14:15:04.737 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:04.738 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2024-09-13T14:15:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:04 smithi189 ceph-mon[33294]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 100 op/s 2024-09-13T14:15:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:04 smithi189 ceph-mon[33294]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:15:04.823 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:04.826 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2024-09-13T14:15:04.913 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:04.916 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:15:04.965 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:04.969 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg 2024-09-13T14:15:04.969 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2024-09-13T14:15:05.010 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_BEFORE=' features: layering 2024-09-13T14:15:05.010 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2024-09-13T14:15:05.010 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:05.052 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:05.055 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_82721/img_v2 testimg_import 2024-09-13T14:15:05.110 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:15:05.114 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg_import 2024-09-13T14:15:05.114 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2024-09-13T14:15:05.153 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_AFTER=' features: layering 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' ' features: layering 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' '!=' ' features: layering 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' ']' 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:05.154 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2024-09-13T14:15:05.223 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:05.225 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2024-09-13T14:15:05.296 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:05.299 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:15:05.360 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:05.363 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:05.404 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:05.408 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_82721/img_v2 testimg_import 2024-09-13T14:15:05.474 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:15:05.478 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2024-09-13T14:15:05.478 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe unit' 2024-09-13T14:15:05.478 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 KiB|4096)' 2024-09-13T14:15:05.518 INFO:tasks.workunit.client.0.smithi017.stdout: stripe unit: 4 KiB 2024-09-13T14:15:05.519 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2024-09-13T14:15:05.519 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe count' 2024-09-13T14:15:05.519 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $3}' 2024-09-13T14:15:05.519 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 1000 2024-09-13T14:15:05.558 INFO:tasks.workunit.client.0.smithi017.stdout:1000 2024-09-13T14:15:05.558 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/img_v2 2024-09-13T14:15:05.558 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2024-09-13T14:15:05.649 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T14:15:05.652 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2024-09-13T14:15:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:05 smithi017 ceph-mon[29399]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:15:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:05 smithi017 ceph-mon[38255]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:15:05.743 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-13T14:15:05.747 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_82721/img testimg 2024-09-13T14:15:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:05 smithi189 ceph-mon[33294]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:15:05.808 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:05.811 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap1 2024-09-13T14:15:06.431 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:15:06.437 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap2 2024-09-13T14:15:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:06 smithi017 ceph-mon[29399]: pgmap v102: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2024-09-13T14:15:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:06 smithi017 ceph-mon[38255]: pgmap v102: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2024-09-13T14:15:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:06 smithi189 ceph-mon[33294]: pgmap v102: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 218 op/s 2024-09-13T14:15:07.140 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:07.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:07 smithi017 ceph-mon[29399]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:15:07.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:07 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3321468429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:07.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:07 smithi017 ceph-mon[38255]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:15:07.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:07 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3321468429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:07.435 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-09-13T14:15:07.443 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap protect testimg@snap2 2024-09-13T14:15:07.491 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_82721/snap_protect 2024-09-13T14:15:07.534 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:07.538 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_82721/snap_protect testimg_import 2024-09-13T14:15:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:07 smithi189 ceph-mon[33294]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:15:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:07 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3321468429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:08.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:07] "GET /metrics HTTP/1.1" 200 239771 "" "Prometheus/2.43.0" 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 177 op/s 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:15:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 177 op/s 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:15:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: pgmap v104: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 12 KiB/s wr, 177 op/s 2024-09-13T14:15:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:15:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:15:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:08.816 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-09-13T14:15:08.819 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap1 2024-09-13T14:15:08.820 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: False' 2024-09-13T14:15:08.860 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2024-09-13T14:15:08.861 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap2 2024-09-13T14:15:08.861 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: True' 2024-09-13T14:15:08.902 INFO:tasks.workunit.client.0.smithi017.stdout: protected: True 2024-09-13T14:15:08.902 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/snap_protect 2024-09-13T14:15:08.902 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg@snap2 2024-09-13T14:15:08.951 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg_import@snap2 2024-09-13T14:15:08.999 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg 2024-09-13T14:15:09.798 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:10.148 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:09 smithi017 ceph-mon[29399]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:15:10.148 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:09 smithi017 ceph-mon[29399]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 449 KiB/s rd, 52 KiB/s wr, 601 op/s 2024-09-13T14:15:10.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:09 smithi017 ceph-mon[38255]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:15:10.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:09 smithi017 ceph-mon[38255]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 449 KiB/s rd, 52 KiB/s wr, 601 op/s 2024-09-13T14:15:10.149 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:10.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:09 smithi189 ceph-mon[33294]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:15:10.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:09 smithi189 ceph-mon[33294]: pgmap v108: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 449 KiB/s rd, 52 KiB/s wr, 601 op/s 2024-09-13T14:15:10.810 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T14:15:10.817 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg_import 2024-09-13T14:15:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[38255]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:15:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2639501150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[38255]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[38255]: from='client.25067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[29399]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:15:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2639501150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[29399]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:10 smithi017 ceph-mon[29399]: from='client.25067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:10 smithi189 ceph-mon[33294]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:15:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:10 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2639501150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:10 smithi189 ceph-mon[33294]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:10 smithi189 ceph-mon[33294]: from='client.25067 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:11 smithi017 ceph-mon[29399]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:15:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:11 smithi017 ceph-mon[29399]: pgmap v111: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 53 KiB/s wr, 616 op/s 2024-09-13T14:15:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:11 smithi017 ceph-mon[38255]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:15:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:11 smithi017 ceph-mon[38255]: pgmap v111: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 53 KiB/s wr, 616 op/s 2024-09-13T14:15:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:11 smithi189 ceph-mon[33294]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:15:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:11 smithi189 ceph-mon[33294]: pgmap v111: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 53 KiB/s wr, 616 op/s 2024-09-13T14:15:12.801 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-13T14:15:12.809 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2024-09-13T14:15:12.908 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:12.911 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2024-09-13T14:15:13.009 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:13.013 INFO:tasks.workunit.client.0.smithi017.stderr:+ tiered=0 2024-09-13T14:15:13.013 INFO:tasks.workunit.client.0.smithi017.stderr:+ ceph osd dump 2024-09-13T14:15:13.013 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep '^pool' 2024-09-13T14:15:13.013 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep ''\''rbd'\''' 2024-09-13T14:15:13.013 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep tier 2024-09-13T14:15:13.168 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:13.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:12 smithi017 ceph-mon[29399]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:15:13.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:12 smithi017 ceph-mon[29399]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:15:13.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:12 smithi017 ceph-mon[38255]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:15:13.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:12 smithi017 ceph-mon[38255]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:15:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:12 smithi189 ceph-mon[33294]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:15:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:12 smithi189 ceph-mon[33294]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:15:13.442 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_82721/sparse1 2024-09-13T14:15:13.446 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2024-09-13T14:15:13.446 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2024-09-13T14:15:13.447 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0033977 s, 309 MB/s 2024-09-13T14:15:13.447 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_82721/sparse2 2024-09-13T14:15:13.451 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2024-09-13T14:15:13.451 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2024-09-13T14:15:13.451 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0035877 s, 292 MB/s 2024-09-13T14:15:13.451 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_82721/sparse2 -s 2M 2024-09-13T14:15:13.455 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2024-09-13T14:15:13.498 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2024-09-13T14:15:13.498 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:15:13.501 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:15:13.501 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_82721/sparse1 2024-09-13T14:15:13.569 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T14:15:13.570 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:13.570 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:13.573 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2024-09-13T14:15:13.573 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2024-09-13T14:15:13.574 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-13T14:15:13.615 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 2 MiB 2 2024-09-13T14:15:13.615 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2024-09-13T14:15:13.615 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2024-09-13T14:15:13.616 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2024-09-13T14:15:13.616 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:13.616 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:13.657 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.62212d67f8bc 2024-09-13T14:15:13.658 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.62212d67f8bc 2024-09-13T14:15:13.658 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2024-09-13T14:15:13.658 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2024-09-13T14:15:13.658 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.62212d67f8bc\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:13.658 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:13.659 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2024-09-13T14:15:13.659 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:13.659 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:13.699 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:13.699 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:13.699 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:13.699 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:13.699 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:13.735 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2024-09-13T14:15:13.735 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2024-09-13T14:15:13.735 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:15:13.735 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:13.775 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:13.779 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:13.779 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:13.781 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse1 --format %b 2024-09-13T14:15:13.782 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2024-09-13T14:15:13.782 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse1.out --format %b 2024-09-13T14:15:13.783 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2024-09-13T14:15:13.783 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:13.784 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:13.784 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:15:13.784 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:13.784 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2024-09-13T14:15:13.881 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T14:15:13.885 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2024-09-13T14:15:13.924 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2024-09-13T14:15:13.924 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2024-09-13T14:15:13.927 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:15:13.927 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_82721/sparse2 2024-09-13T14:15:13.989 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-09-13T14:15:13.990 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:13.990 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:13.993 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2024-09-13T14:15:13.993 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2024-09-13T14:15:13.993 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-09-13T14:15:14.030 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 2 MiB 2 2024-09-13T14:15:14.031 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2024-09-13T14:15:14.031 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2024-09-13T14:15:14.031 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2024-09-13T14:15:14.032 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:14.032 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:14.072 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3ba017d2695b 2024-09-13T14:15:14.072 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3ba017d2695b 2024-09-13T14:15:14.072 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3ba017d2695b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:14.073 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:14.114 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:14.115 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:14.115 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:14.115 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:14.115 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:14.150 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2024-09-13T14:15:14.150 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2024-09-13T14:15:14.150 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-13T14:15:14.150 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.192 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:14.196 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.196 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.198 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse2 --format %b 2024-09-13T14:15:14.199 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2024-09-13T14:15:14.199 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse2.out --format %b 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2342085033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[29399]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2342085033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:14.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:13 smithi017 ceph-mon[38255]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.202 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2024-09-13T14:15:14.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2342085033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:14.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: pgmap v114: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3017536575' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-09-13T14:15:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:13 smithi189 ceph-mon[33294]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:14.288 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-09-13T14:15:14.291 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_82721/sparse1 -s 10M 2024-09-13T14:15:14.292 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse1 2024-09-13T14:15:14.380 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:14.381 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:14.381 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:14.384 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2024-09-13T14:15:14.384 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2024-09-13T14:15:14.384 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-09-13T14:15:14.422 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 10 MiB 2 2024-09-13T14:15:14.423 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2024-09-13T14:15:14.423 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2024-09-13T14:15:14.423 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2024-09-13T14:15:14.423 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:14.423 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3bcd3b498b6b 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3bcd3b498b6b 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3bcd3b498b6b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:14.465 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:14.504 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:14.504 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:14.504 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:14.504 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:14.504 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:14.540 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2024-09-13T14:15:14.540 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2024-09-13T14:15:14.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:15:14.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:14.582 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:14.586 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:14.586 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/sparse1 /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:14.594 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse1 --format %b 2024-09-13T14:15:14.595 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2024-09-13T14:15:14.595 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse1.out --format %b 2024-09-13T14:15:14.596 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2024-09-13T14:15:14.596 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.597 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.597 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:15:14.597 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse1.out 2024-09-13T14:15:14.598 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2024-09-13T14:15:14.684 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T14:15:14.687 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_82721/sparse2 oflag=append conv=notrunc 2024-09-13T14:15:14.694 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2024-09-13T14:15:14.694 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2024-09-13T14:15:14.694 INFO:tasks.workunit.client.0.smithi017.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00648408 s, 323 MB/s 2024-09-13T14:15:14.695 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse2 2024-09-13T14:15:14.772 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:14.773 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:14.773 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:14.776 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2024-09-13T14:15:14.776 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2024-09-13T14:15:14.776 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-09-13T14:15:14.817 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 4 MiB 2 2024-09-13T14:15:14.818 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2024-09-13T14:15:14.818 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2024-09-13T14:15:14.818 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2024-09-13T14:15:14.818 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:14.818 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:14.858 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.62aea4d7d5ed 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.62aea4d7d5ed 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.62aea4d7d5ed\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:14.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:14.898 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:14.898 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:14.898 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:14.899 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:14.899 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:14.932 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects='0 2024-09-13T14:15:14.933 INFO:tasks.workunit.client.0.smithi017.stderr:2 2024-09-13T14:15:14.933 INFO:tasks.workunit.client.0.smithi017.stderr:3' 2024-09-13T14:15:14.933 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2 3 2024-09-13T14:15:14.933 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-09-13T14:15:14.933 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.975 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-09-13T14:15:14.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_82721/sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_82721/sparse2 /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.982 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse2 --format %b 2024-09-13T14:15:14.983 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=6144 2024-09-13T14:15:14.983 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse2.out --format %b 2024-09-13T14:15:14.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=6144 2024-09-13T14:15:14.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2024-09-13T14:15:14.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-09-13T14:15:14.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse2.out 2024-09-13T14:15:14.985 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2024-09-13T14:15:15.072 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-09-13T14:15:15.075 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-09-13T14:15:15.075 INFO:tasks.workunit.client.0.smithi017.stdout:partially-sparse file imports to partially-sparse image 2024-09-13T14:15:15.076 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_82721/sparse1 sparse 2024-09-13T14:15:15.142 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-09-13T14:15:15.142 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:15.142 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2024-09-13T14:15:15.145 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2024-09-13T14:15:15.145 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2024-09-13T14:15:15.146 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2024-09-13T14:15:15.146 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:15.146 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:15.185 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3c2763f32611 2024-09-13T14:15:15.186 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3c2763f32611 2024-09-13T14:15:15.186 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2024-09-13T14:15:15.186 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3c2763f32611\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:15.186 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2024-09-13T14:15:15.186 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:15.187 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2024-09-13T14:15:15.187 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:15.187 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:15.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:15.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:15.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:15.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:15.226 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:15.262 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2024-09-13T14:15:15.262 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2024-09-13T14:15:15.262 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-09-13T14:15:15.262 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2024-09-13T14:15:15.350 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-09-13T14:15:15.353 INFO:tasks.workunit.client.0.smithi017.stdout:zeros import through stdin to sparse image 2024-09-13T14:15:15.354 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros import through stdin to sparse image' 2024-09-13T14:15:15.354 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=1M count=4 2024-09-13T14:15:15.354 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - sparse 2024-09-13T14:15:15.406 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records in 2024-09-13T14:15:15.407 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records out 2024-09-13T14:15:15.407 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0519093 s, 80.8 MB/s 2024-09-13T14:15:15.416 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2024-09-13T14:15:15.417 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:15.420 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2024-09-13T14:15:15.420 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2024-09-13T14:15:15.420 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2024-09-13T14:15:15.420 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:15.420 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:15.459 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.62fcdbb2e6d6 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.62fcdbb2e6d6 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.62fcdbb2e6d6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:15.460 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2024-09-13T14:15:15.461 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:15.461 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:15.502 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:15.502 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:15.502 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:15.502 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:15.502 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:15.540 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects= 2024-09-13T14:15:15.540 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 2024-09-13T14:15:15.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-09-13T14:15:15.541 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2024-09-13T14:15:15.630 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:15.633 INFO:tasks.workunit.client.0.smithi017.stdout:zeros export to sparse file 2024-09-13T14:15:15.634 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros export to sparse file' 2024-09-13T14:15:15.634 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd create --new-format sparse --size 4 2024-09-13T14:15:15.677 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-09-13T14:15:15.680 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info sparse 2024-09-13T14:15:15.680 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep block_name_prefix 2024-09-13T14:15:15.681 INFO:tasks.workunit.client.0.smithi017.stderr:++ awk '{print $NF;}' 2024-09-13T14:15:15.722 INFO:tasks.workunit.client.0.smithi017.stderr:+ prefix=rbd_data.3c6fd61348f1 2024-09-13T14:15:15.722 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=4M count=1 2024-09-13T14:15:15.723 INFO:tasks.workunit.client.0.smithi017.stderr:++ get_image_data_pool sparse 2024-09-13T14:15:15.723 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2024-09-13T14:15:15.723 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2024-09-13T14:15:15.723 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep 'data_pool: ' 2024-09-13T14:15:15.723 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk -F: '{ print $NF }' 2024-09-13T14:15:15.765 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool= 2024-09-13T14:15:15.766 INFO:tasks.workunit.client.0.smithi017.stderr:++ '[' -z ']' 2024-09-13T14:15:15.766 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool=rbd 2024-09-13T14:15:15.766 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo rbd 2024-09-13T14:15:15.766 INFO:tasks.workunit.client.0.smithi017.stderr:+ rados -p rbd put rbd_data.3c6fd61348f1.000000000000 - 2024-09-13T14:15:15.798 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2024-09-13T14:15:15.798 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2024-09-13T14:15:15.798 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0752983 s, 55.7 MB/s 2024-09-13T14:15:15.816 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2024-09-13T14:15:15.817 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2024-09-13T14:15:15.817 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2024-09-13T14:15:15.817 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2024-09-13T14:15:15.817 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2024-09-13T14:15:15.859 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3c6fd61348f1 2024-09-13T14:15:15.859 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3c6fd61348f1 2024-09-13T14:15:15.859 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2024-09-13T14:15:15.860 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3c6fd61348f1\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-09-13T14:15:15.860 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2024-09-13T14:15:15.860 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2024-09-13T14:15:15.860 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2024-09-13T14:15:15.860 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2024-09-13T14:15:15.861 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2024-09-13T14:15:15.904 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2024-09-13T14:15:15.904 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2024-09-13T14:15:15.905 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2024-09-13T14:15:15.905 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2024-09-13T14:15:15.905 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2024-09-13T14:15:15.952 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2024-09-13T14:15:15.952 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2024-09-13T14:15:15.952 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-09-13T14:15:15.952 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse 2024-09-13T14:15:15.953 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_82721/sparse': No such file or directory 2024-09-13T14:15:15.953 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:15:15.953 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse /tmp/rbd_import_export_82721/sparse 2024-09-13T14:15:15.995 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2024-09-13T14:15:15.999 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_82721/sparse --format=%b 2024-09-13T14:15:16.000 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 = 0 ']' 2024-09-13T14:15:16.000 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2024-09-13T14:15:16.086 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2024-09-13T14:15:16.089 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_82721/sparse /tmp/rbd_import_export_82721/sparse1 /tmp/rbd_import_export_82721/sparse2 /tmp/rbd_import_export_82721/sparse3 2024-09-13T14:15:16.092 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_82721/sparse3': No such file or directory 2024-09-13T14:15:16.092 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2024-09-13T14:15:16.092 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo OK 2024-09-13T14:15:16.092 INFO:tasks.workunit.client.0.smithi017.stdout:OK 2024-09-13T14:15:16.093 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_82721 2024-09-13T14:15:16.093 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:15:16.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:15:16.126 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-09-13T14:15:16.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:15:16.231 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:16.416 DEBUG:teuthology.parallel:result is None 2024-09-13T14:15:16.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:15:16.445 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:15:16.445 DEBUG:teuthology.orchestra.run.smithi017:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:15:16.502 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:15:16.502 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:15:16.502 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-09-13T14:15:16.502 DEBUG:teuthology.parallel:result is None 2024-09-13T14:15:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:16 smithi017 ceph-mon[29399]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 381 KiB/s wr, 205 op/s 2024-09-13T14:15:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:16 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2674691915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:16 smithi017 ceph-mon[38255]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 381 KiB/s wr, 205 op/s 2024-09-13T14:15:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:16 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2674691915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:16 smithi189 ceph-mon[33294]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 381 KiB/s wr, 205 op/s 2024-09-13T14:15:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:16 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2674691915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:17 smithi017 ceph-mon[38255]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:17 smithi017 ceph-mon[38255]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:17 smithi017 ceph-mon[29399]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:17 smithi017 ceph-mon[29399]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:17 smithi189 ceph-mon[33294]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:17 smithi189 ceph-mon[33294]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:18.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:17] "GET /metrics HTTP/1.1" 200 240300 "" "Prometheus/2.43.0" 2024-09-13T14:15:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:18 smithi017 ceph-mon[29399]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 443 KiB/s rd, 323 KiB/s wr, 174 op/s 2024-09-13T14:15:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:18 smithi017 ceph-mon[29399]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:15:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:18 smithi017 ceph-mon[38255]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 443 KiB/s rd, 323 KiB/s wr, 174 op/s 2024-09-13T14:15:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:18 smithi017 ceph-mon[38255]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:15:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:18 smithi189 ceph-mon[33294]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 443 KiB/s rd, 323 KiB/s wr, 174 op/s 2024-09-13T14:15:18.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:18 smithi189 ceph-mon[33294]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:15:19.199 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4184356269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:19 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4184356269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:19 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4184356269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:19.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[29399]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[29399]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 367 op/s 2024-09-13T14:15:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[29399]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[38255]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[38255]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 367 op/s 2024-09-13T14:15:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:20 smithi017 ceph-mon[38255]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:20 smithi189 ceph-mon[33294]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:20 smithi189 ceph-mon[33294]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 367 op/s 2024-09-13T14:15:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:20 smithi189 ceph-mon[33294]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:22.255 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:22 smithi017 ceph-mon[29399]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 961 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2024-09-13T14:15:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:22 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1490540137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:22 smithi017 ceph-mon[38255]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 961 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2024-09-13T14:15:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:22 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1490540137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:22 smithi189 ceph-mon[33294]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 961 KiB/s rd, 1.3 MiB/s wr, 323 op/s 2024-09-13T14:15:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:22 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1490540137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[29399]: from='client.25475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[29399]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[38255]: from='client.25475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[38255]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:23 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:23 smithi189 ceph-mon[33294]: from='client.25475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:23 smithi189 ceph-mon[33294]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:23 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:24.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:24 smithi017 ceph-mon[29399]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 820 KiB/s rd, 1.1 MiB/s wr, 275 op/s 2024-09-13T14:15:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:24 smithi017 ceph-mon[38255]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 820 KiB/s rd, 1.1 MiB/s wr, 275 op/s 2024-09-13T14:15:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:24 smithi189 ceph-mon[33294]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 820 KiB/s rd, 1.1 MiB/s wr, 275 op/s 2024-09-13T14:15:25.317 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:25 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3953254805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:25 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3953254805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:25 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3953254805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[29399]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[29399]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[29399]: from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[38255]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[38255]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:26.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:26 smithi017 ceph-mon[38255]: from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:26 smithi189 ceph-mon[33294]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:26 smithi189 ceph-mon[33294]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:26 smithi189 ceph-mon[33294]: from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:28.198 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:27] "GET /metrics HTTP/1.1" 200 240552 "" "Prometheus/2.43.0" 2024-09-13T14:15:28.198 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:28 smithi017 ceph-mon[38255]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:28 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1922582701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:28 smithi017 ceph-mon[29399]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:28 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1922582701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:28 smithi189 ceph-mon[33294]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 531 KiB/s rd, 929 KiB/s wr, 162 op/s 2024-09-13T14:15:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:28 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1922582701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[29399]: from='client.25511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[29399]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[38255]: from='client.25511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[38255]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:29 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:29.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-mon[33294]: from='client.25511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-mon[33294]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:29 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[29399]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[29399]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[29399]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[29399]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi189 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[29399]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 805 KiB/s wr, 140 op/s 2024-09-13T14:15:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[38255]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:15:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[38255]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:15:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[38255]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:15:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[38255]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi189 2024-09-13T14:15:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:30 smithi017 ceph-mon[38255]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 805 KiB/s wr, 140 op/s 2024-09-13T14:15:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:30 smithi189 ceph-mon[33294]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:15:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:30 smithi189 ceph-mon[33294]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:15:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:30 smithi189 ceph-mon[33294]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:15:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:30 smithi189 ceph-mon[33294]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi189 2024-09-13T14:15:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:30 smithi189 ceph-mon[33294]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 461 KiB/s rd, 805 KiB/s wr, 140 op/s 2024-09-13T14:15:31.072 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:31 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2998008791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:31 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2998008791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:31 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2998008791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[29399]: from='client.25529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[29399]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[29399]: from='client.25535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[38255]: from='client.25529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[38255]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:32 smithi017 ceph-mon[38255]: from='client.25535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:32 smithi189 ceph-mon[33294]: from='client.25529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:32 smithi189 ceph-mon[33294]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:32 smithi189 ceph-mon[33294]: from='client.25535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:33.943 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:34 smithi017 ceph-mon[29399]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:34 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1638264651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:34 smithi017 ceph-mon[38255]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:34 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1638264651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:34 smithi189 ceph-mon[33294]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:34 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1638264651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:35 smithi017 ceph-mon[29399]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:35 smithi017 ceph-mon[29399]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:35 smithi017 ceph-mon[38255]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:35 smithi017 ceph-mon[38255]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:35 smithi189 ceph-mon[33294]: from='client.15639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:35 smithi189 ceph-mon[33294]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:36 smithi017 ceph-mon[29399]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:36 smithi017 ceph-mon[38255]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:36 smithi189 ceph-mon[33294]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:36.818 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2474767155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2474767155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2474767155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:38.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:37] "GET /metrics HTTP/1.1" 200 240552 "" "Prometheus/2.43.0" 2024-09-13T14:15:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[29399]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[29399]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[29399]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[38255]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[38255]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[38255]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:38 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:38 smithi189 ceph-mon[33294]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:38 smithi189 ceph-mon[33294]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:38.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:38 smithi189 ceph-mon[33294]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:38.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:38 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:39.691 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:39.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:39 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2771784041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:39.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:39 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2771784041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:39 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2771784041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:39.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:40 smithi017 ceph-mon[29399]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:40 smithi017 ceph-mon[38255]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:40 smithi189 ceph-mon[33294]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:41 smithi017 ceph-mon[29399]: from='client.25583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:41 smithi017 ceph-mon[29399]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:41 smithi017 ceph-mon[38255]: from='client.25583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:41 smithi017 ceph-mon[38255]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:41 smithi189 ceph-mon[33294]: from='client.25583 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:41 smithi189 ceph-mon[33294]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:42.567 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:42 smithi017 ceph-mon[29399]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:42 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3747662156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:42 smithi017 ceph-mon[38255]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:42 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3747662156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:42 smithi189 ceph-mon[33294]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:42 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3747662156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:43 smithi017 ceph-mon[29399]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:43 smithi017 ceph-mon[38255]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:43 smithi189 ceph-mon[33294]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:44 smithi017 ceph-mon[29399]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:44 smithi017 ceph-mon[29399]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:44 smithi017 ceph-mon[38255]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:44 smithi017 ceph-mon[38255]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:44 smithi189 ceph-mon[33294]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:44 smithi189 ceph-mon[33294]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:45.439 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:45 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/916876233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:45 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/916876233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:45 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/916876233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:46 smithi017 ceph-mon[38255]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:46 smithi017 ceph-mon[38255]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:46 smithi017 ceph-mon[29399]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:46 smithi017 ceph-mon[29399]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:46 smithi189 ceph-mon[33294]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:46 smithi189 ceph-mon[33294]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:47 smithi189 ceph-mon[33294]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:47.862 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:47 smithi017 ceph-mon[29399]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:47.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:47 smithi017 ceph-mon[38255]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:48.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:47] "GET /metrics HTTP/1.1" 200 240493 "" "Prometheus/2.43.0" 2024-09-13T14:15:48.315 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:48.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:48 smithi017 ceph-mon[38255]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:48.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:48 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3037631992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:48.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:48 smithi017 ceph-mon[29399]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:48.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:48 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3037631992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:48 smithi189 ceph-mon[33294]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:48 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3037631992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:49 smithi189 ceph-mon[33294]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:49 smithi189 ceph-mon[33294]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:49.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:15:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:49 smithi017 ceph-mon[38255]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:49 smithi017 ceph-mon[38255]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:49 smithi017 ceph-mon[29399]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:49 smithi017 ceph-mon[29399]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:50 smithi189 ceph-mon[33294]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:50 smithi017 ceph-mon[29399]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:50 smithi017 ceph-mon[38255]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:51.200 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:51 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/309543851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:51 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/309543851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:51 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/309543851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:52 smithi189 ceph-mon[33294]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:52 smithi189 ceph-mon[33294]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:52 smithi189 ceph-mon[33294]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:52.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[29399]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:52.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[29399]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:52.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[29399]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:52.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[38255]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:52.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[38255]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:52.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:52 smithi017 ceph-mon[38255]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:53 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:53 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:53.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:53 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:15:54.077 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:54 smithi189 ceph-mon[33294]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:54 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1174995607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:54 smithi017 ceph-mon[38255]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:54 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1174995607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:54 smithi017 ceph-mon[29399]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:54 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1174995607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:55 smithi189 ceph-mon[33294]: from='client.25673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:55 smithi189 ceph-mon[33294]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:55 smithi017 ceph-mon[29399]: from='client.25673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:55 smithi017 ceph-mon[29399]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:55.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:55 smithi017 ceph-mon[38255]: from='client.25673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:55.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:55 smithi017 ceph-mon[38255]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:56 smithi189 ceph-mon[33294]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:56.949 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:56.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:56 smithi017 ceph-mon[29399]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:56 smithi017 ceph-mon[38255]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:57 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/782031247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:57.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:57 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/782031247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:57.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:57 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/782031247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:15:58.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:15:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:57] "GET /metrics HTTP/1.1" 200 240429 "" "Prometheus/2.43.0" 2024-09-13T14:15:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:58 smithi189 ceph-mon[33294]: from='client.25691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:58 smithi189 ceph-mon[33294]: from='client.25697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:15:58 smithi189 ceph-mon[33294]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[29399]: from='client.25691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[29399]: from='client.25697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[29399]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[38255]: from='client.25691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[38255]: from='client.25697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:15:58 smithi017 ceph-mon[38255]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:15:59.833 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:15:59.833 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:15:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[35335]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:15:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4288501954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:16:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:16:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:00 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4288501954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4288501954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:16:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:00 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:01.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:01 smithi189 ceph-mon[33294]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:01 smithi189 ceph-mon[33294]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:01 smithi189 ceph-mon[33294]: Upgrade: Updating mgr.x 2024-09-13T14:16:01.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:01 smithi189 ceph-mon[33294]: Deploying daemon mgr.x on smithi189 2024-09-13T14:16:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[29399]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[29399]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[29399]: Upgrade: Updating mgr.x 2024-09-13T14:16:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[29399]: Deploying daemon mgr.x on smithi189 2024-09-13T14:16:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[38255]: from='client.15801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[38255]: from='client.15807 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[38255]: Upgrade: Updating mgr.x 2024-09-13T14:16:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:01 smithi017 ceph-mon[38255]: Deploying daemon mgr.x on smithi189 2024-09-13T14:16:02.005 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:01 smithi189 systemd[1]: Stopping Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:02.490 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:02 smithi189 podman[68654]: 2024-09-13 14:16:02.234041965 +0000 UTC m=+0.125870682 container died 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:16:02.708 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:02.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:02 smithi189 podman[68654]: 2024-09-13 14:16:02.711209325 +0000 UTC m=+0.603037996 container cleanup 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:16:02.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:02 smithi189 bash[68654]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:16:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:02 smithi189 ceph-mon[33294]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:02 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/25479779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:02 smithi017 ceph-mon[29399]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:02 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/25479779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:02 smithi017 ceph-mon[38255]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:02 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/25479779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:03.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:02 smithi189 podman[68672]: 2024-09-13 14:16:02.96290215 +0000 UTC m=+0.721165947 container remove 0e0fb3e7d21141de8bfda89875a9363d233df660f92639ee42e749698236b4e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T14:16:03.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:02 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:16:03.256 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:16:03.256 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 systemd[1]: Stopped Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:03.256 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Consumed 10.204s CPU time. 2024-09-13T14:16:03.702 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 systemd[1]: Starting Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:03.702 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 podman[68807]: 2024-09-13 14:16:03.4701684 +0000 UTC m=+0.016033192 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:16:03.703 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 podman[68807]: 2024-09-13 14:16:03.58889855 +0000 UTC m=+0.134763335 container create ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:16:04.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 podman[68807]: 2024-09-13 14:16:03.816337443 +0000 UTC m=+0.362202221 container init ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:16:04.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 podman[68807]: 2024-09-13 14:16:03.821207425 +0000 UTC m=+0.367072216 container start ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:16:04.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 ceph-mgr[68845]: -- 172.21.15.189:0/1395435703 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562e743ef0e0 con 0x562e743cc800 2024-09-13T14:16:04.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 bash[68807]: ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb 2024-09-13T14:16:04.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:03 smithi189 systemd[1]: Started Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:04.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.753 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:04.753 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.753 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.753 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:04 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:04.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:04.756 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:04 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:05.612 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:05 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3708855926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:05.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:05 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3708855926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:05.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:05 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3708855926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:06 smithi017 ceph-mon[29399]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:06 smithi017 ceph-mon[29399]: from='client.15837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:06 smithi017 ceph-mon[38255]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:06 smithi017 ceph-mon[38255]: from='client.15837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:06 smithi189 ceph-mon[33294]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:06 smithi189 ceph-mon[33294]: from='client.15837 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:07 smithi189 ceph-mon[33294]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:07 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:07 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:07.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[29399]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:07.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:07.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:07.863 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[38255]: from='client.15843 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:07.863 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:07.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:07] "GET /metrics HTTP/1.1" 200 240429 "" "Prometheus/2.43.0" 2024-09-13T14:16:08.486 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:08 smithi189 ceph-mon[33294]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:08 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1377723679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:08 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[29399]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1377723679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[38255]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1377723679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:08 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:09] ENGINE Bus STARTING 2024-09-13T14:16:09.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: CherryPy Checker: 2024-09-13T14:16:09.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: The Application mounted at '' has an empty config. 2024-09-13T14:16:09.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:09.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:09] ENGINE Serving on http://:::9283 2024-09-13T14:16:09.504 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:09] ENGINE Bus STARTED 2024-09-13T14:16:09.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='client.25763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: Standby manager daemon x restarted 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:16:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:09.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:09.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:09.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:09 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='client.25763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: Standby manager daemon x restarted 2024-09-13T14:16:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: Standby manager daemon x started 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='client.25763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: Standby manager daemon x restarted 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:16:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:09.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:16:09.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:09 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:10.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:10 smithi189 ceph-mon[33294]: from='client.25769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:10.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:10 smithi189 ceph-mon[33294]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:10 smithi189 ceph-mon[33294]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[29399]: from='client.25769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[29399]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[29399]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[38255]: from='client.25769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[38255]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:10 smithi017 ceph-mon[38255]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-13T14:16:11.368 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:11 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3360987414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:11 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3360987414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:12.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:11 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3360987414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:12.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:12] ENGINE Bus STOPPING 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:12 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:12.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:12.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:12.604+0000 7f98bd26d640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setuser ceph since I am not root 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ignoring --setgroup ceph since I am not root 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-mgr[29672]: -- 172.21.15.17:0/3517184873 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d315aca340 con 0x55d315ad7000 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:12.755+0000 7f5a91e05ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:16:12.820 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:12.816+0000 7f5a91e05ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:13.165 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:12.889+0000 7f5a91e05ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:16:13.166 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:13.018+0000 7f5a91e05ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:13.433 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:13.164+0000 7f5a91e05ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:16:13.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:16:13.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:13] ENGINE Bus STOPPED 2024-09-13T14:16:13.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:13.432+0000 7f5a91e05ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:16:13.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:13.495+0000 7f5a91e05ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:16:13.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:13.670+0000 7f5a91e05ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: Activating manager daemon x 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: mgrmap e23: x(active, starting, since 0.00718756s) 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[38255]: Manager daemon x is now available 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: Activating manager daemon x 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: mgrmap e23: x(active, starting, since 0.00718756s) 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:13.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:13.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:13 smithi017 ceph-mon[29399]: Manager daemon x is now available 2024-09-13T14:16:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: Activating manager daemon x 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.14412 172.21.15.17:0/3272024556' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: mgrmap e23: x(active, starting, since 0.00718756s) 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:16:14.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:13 smithi189 ceph-mon[33294]: Manager daemon x is now available 2024-09-13T14:16:14.393 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:14.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:14] ENGINE Bus STARTING 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f9ff2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f9ff2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f9ff2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f9ff2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f9ff2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f67eb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f67eb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f67eb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f67eb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:14.075+0000 7fa0f67eb640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:14] ENGINE Serving on http://:::9283 2024-09-13T14:16:14.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:16:14] ENGINE Bus STARTED 2024-09-13T14:16:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:14.604+0000 7f5a91e05ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:16:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:14.675+0000 7f5a91e05ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:16:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: mgrmap e24: x(active, since 1.00963s) 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3806007822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: mgrmap e24: x(active, since 1.00963s) 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3806007822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:14 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: mgrmap e24: x(active, since 1.00963s) 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3806007822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:14 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:15.099+0000 7fa0e2d4a640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:16:15] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: Traceback (most recent call last): 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: self._connections.run(self.expiration_interval) 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: self._run(expiration_interval) 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:15.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: s = self.context.wrap_socket( 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: return self.sslsocket_class._create( 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: self.do_handshake() 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: self._sslobj.do_handshake() 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:15.507 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: Checking for cert/key for grafana.a 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: Migrating grafana.a cert to cert store 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: Migrating grafana.a key to cert store 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: from='client.15918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: Deploying cephadm binary to smithi189 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: Deploying cephadm binary to smithi017 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[38255]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.419+0000 7f5a91e05ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.536+0000 7f5a91e05ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.664+0000 7f5a91e05ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: Checking for cert/key for grafana.a 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: Migrating grafana.a cert to cert store 2024-09-13T14:16:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: Migrating grafana.a key to cert store 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: from='client.15918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: Deploying cephadm binary to smithi189 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: Deploying cephadm binary to smithi017 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:15.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-mon[29399]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:15.989 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.726+0000 7f5a91e05ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:16:15.989 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.857+0000 7f5a91e05ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:16:15.989 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.928+0000 7f5a91e05ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:16:15.990 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:15 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:15.988+0000 7f5a91e05ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: Checking for cert/key for grafana.a 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: Migrating grafana.a cert to cert store 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: Migrating grafana.a key to cert store 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: from='client.15918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: Deploying cephadm binary to smithi189 2024-09-13T14:16:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: Deploying cephadm binary to smithi017 2024-09-13T14:16:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:15 smithi189 ceph-mon[33294]: from='client.25841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:16.271 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.132+0000 7f5a91e05ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:16:16.271 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.200+0000 7f5a91e05ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:16:16.271 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.269+0000 7f5a91e05ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:16:16.615 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.363+0000 7f5a91e05ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:16:16.615 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.436+0000 7f5a91e05ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:16:16.615 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:16.507+0000 7f5a91e05ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: [13/Sep/2024:14:16:14] ENGINE Bus STARTING 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: [13/Sep/2024:14:16:15] ENGINE Serving on https://172.21.15.189:7150 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: [13/Sep/2024:14:16:15] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:16:16.916 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:16.917 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: [13/Sep/2024:14:16:15] ENGINE Serving on http://172.21.15.189:8765 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: [13/Sep/2024:14:16:15] ENGINE Bus STARTED 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[29399]: mgrmap e25: x(active, since 3s) 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: [13/Sep/2024:14:16:14] ENGINE Bus STARTING 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: [13/Sep/2024:14:16:15] ENGINE Serving on https://172.21.15.189:7150 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: [13/Sep/2024:14:16:15] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:16.918 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:16.919 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: [13/Sep/2024:14:16:15] ENGINE Serving on http://172.21.15.189:8765 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: [13/Sep/2024:14:16:15] ENGINE Bus STARTED 2024-09-13T14:16:16.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:16 smithi017 ceph-mon[38255]: mgrmap e25: x(active, since 3s) 2024-09-13T14:16:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: [13/Sep/2024:14:16:14] ENGINE Bus STARTING 2024-09-13T14:16:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: [13/Sep/2024:14:16:15] ENGINE Serving on https://172.21.15.189:7150 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: [13/Sep/2024:14:16:15] ENGINE Error in HTTPServer.serve 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: Traceback (most recent call last): 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._run(expiration_interval) 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: s = self.context.wrap_socket( 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:16:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: return self.sslsocket_class._create( 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: self.do_handshake() 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._sslobj.do_handshake() 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: [13/Sep/2024:14:16:15] ENGINE Serving on http://172.21.15.189:8765 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: [13/Sep/2024:14:16:15] ENGINE Bus STARTED 2024-09-13T14:16:17.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:16 smithi189 ceph-mon[33294]: mgrmap e25: x(active, since 3s) 2024-09-13T14:16:17.393 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:17.401+0000 7f5a91e05ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:16:17] ENGINE Bus STARTING 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: CherryPy Checker: 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: The Application mounted at '' has an empty config. 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:16:17] ENGINE Serving on http://:::9283 2024-09-13T14:16:17.667 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: [13/Sep/2024:14:16:17] ENGINE Bus STARTED 2024-09-13T14:16:17.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3954271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: Standby manager daemon y started 2024-09-13T14:16:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3954271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: Standby manager daemon y started 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:16:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:17 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3954271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: Standby manager daemon y started 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:16:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:16:18.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:17 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/3560054456' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:16:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[29399]: from='client.25859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[29399]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-13T14:16:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[29399]: from='client.25865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[38255]: from='client.25859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[38255]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-13T14:16:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:18 smithi017 ceph-mon[38255]: from='client.25865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:18 smithi189 ceph-mon[33294]: from='client.25859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:18 smithi189 ceph-mon[33294]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-13T14:16:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:18 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:16:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:18 smithi189 ceph-mon[33294]: from='client.25865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[38255]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 614 B/s wr, 0 op/s 2024-09-13T14:16:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[29399]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 614 B/s wr, 0 op/s 2024-09-13T14:16:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:19 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:19 smithi189 ceph-mon[33294]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 614 B/s wr, 0 op/s 2024-09-13T14:16:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:19 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:19 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:20.006 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:19 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:19] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.43.0" 2024-09-13T14:16:20.366 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:20 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/845548408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/845548408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:21.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:20 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/845548408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:21 smithi189 ceph-mon[33294]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:21 smithi189 ceph-mon[33294]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:16:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:21 smithi189 ceph-mon[33294]: from='client.15972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:22.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[38255]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:22.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[38255]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:16:22.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[38255]: from='client.15972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:22.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[29399]: from='client.25877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:22.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[29399]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 438 B/s wr, 10 op/s 2024-09-13T14:16:22.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:21 smithi017 ceph-mon[29399]: from='client.15972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:23.321 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:24.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:23 smithi189 ceph-mon[33294]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:16:24.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:23 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2405403372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:23 smithi017 ceph-mon[29399]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:16:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:23 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2405403372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:23 smithi017 ceph-mon[38255]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-09-13T14:16:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:23 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2405403372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:24 smithi189 ceph-mon[33294]: from='client.25895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:24 smithi189 ceph-mon[33294]: from='client.25901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:25.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:24 smithi017 ceph-mon[38255]: from='client.25895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:25.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:24 smithi017 ceph-mon[38255]: from='client.25901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:24 smithi017 ceph-mon[29399]: from='client.25895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:24 smithi017 ceph-mon[29399]: from='client.25901 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:25 smithi189 ceph-mon[33294]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:16:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:25 smithi017 ceph-mon[29399]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:16:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:25 smithi017 ceph-mon[38255]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-09-13T14:16:26.275 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:26 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3211317688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:26 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3211317688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:26 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3211317688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:28.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:27 smithi189 ceph-mon[33294]: from='client.16002 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:27 smithi189 ceph-mon[33294]: from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:27 smithi189 ceph-mon[33294]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:28.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[38255]: from='client.16002 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[38255]: from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[38255]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[29399]: from='client.16002 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[29399]: from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:27 smithi017 ceph-mon[29399]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:29.221 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:29.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:29] "GET /metrics HTTP/1.1" 200 34003 "" "Prometheus/2.43.0" 2024-09-13T14:16:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[29399]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/425462558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:30.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[38255]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:30.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/425462558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:30.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:29 smithi189 ceph-mon[33294]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-09-13T14:16:30.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:29 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/425462558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:30.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:31.152 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:30 smithi189 ceph-mon[33294]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:31.152 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:30 smithi189 ceph-mon[33294]: from='client.25937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:30 smithi017 ceph-mon[29399]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:30 smithi017 ceph-mon[29399]: from='client.25937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:30 smithi017 ceph-mon[38255]: from='client.16020 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:30 smithi017 ceph-mon[38255]: from='client.25937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:32.163 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[29399]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[38255]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:31 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:31 smithi189 ceph-mon[33294]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:16:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:31 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:31 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.962 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:32 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1014074753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:32.962 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:32 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:32.962 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:32 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1014074753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1014074753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:32 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:34.066 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:33 smithi189 ceph-mon[33294]: from='client.25949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.066 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:33 smithi189 ceph-mon[33294]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.066 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:33 smithi189 ceph-mon[33294]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[29399]: from='client.25949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[29399]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[29399]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[38255]: from='client.25949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[38255]: from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:33 smithi017 ceph-mon[38255]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:35.106 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2769561011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:35 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2769561011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2769561011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:35 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: Adjusting osd_memory_target on smithi189 to 2480M 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: Adjusting osd_memory_target on smithi189 to 2480M 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:36 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='client.16056 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: Adjusting osd_memory_target on smithi189 to 2480M 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:16:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:36 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:38.052 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:38.052 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:38.052 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3353611400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:16:38.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:16:38.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:38.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3353611400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:16:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:16:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:16:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: Deploying daemon alertmanager.a on smithi017 2024-09-13T14:16:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3353611400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:39.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:39] "GET /metrics HTTP/1.1" 200 34003 "" "Prometheus/2.43.0" 2024-09-13T14:16:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[29399]: from='client.16074 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[29399]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[29399]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[38255]: from='client.16074 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[38255]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:39 smithi017 ceph-mon[38255]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:40.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:39 smithi189 ceph-mon[33294]: from='client.16074 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:39 smithi189 ceph-mon[33294]: from='client.16080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:39 smithi189 ceph-mon[33294]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:41.016 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:40 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3118299768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:40 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3118299768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:40 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3118299768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:41 smithi017 ceph-mon[29399]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:41 smithi017 ceph-mon[38255]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:41 smithi189 ceph-mon[33294]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:42.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:42 smithi017 ceph-mon[29399]: from='client.26000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:42.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:42 smithi017 ceph-mon[29399]: from='client.16098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:42.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:42 smithi017 ceph-mon[38255]: from='client.26000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:42.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:42 smithi017 ceph-mon[38255]: from='client.16098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:42 smithi189 ceph-mon[33294]: from='client.26000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:42 smithi189 ceph-mon[33294]: from='client.16098 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:43.975 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:43.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:43 smithi017 ceph-mon[29399]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:43.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:43 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2017832984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:43.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:43 smithi017 ceph-mon[38255]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:43.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:43 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2017832984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:43 smithi189 ceph-mon[33294]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:43 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2017832984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:45.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:44 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:45.142 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:44 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:45.143 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:44 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:45.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:44 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:45.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[74911]: ts=2024-09-13T14:16:45.141Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:16:45.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 podman[91262]: 2024-09-13 14:16:45.182107511 +0000 UTC m=+0.147504128 container died c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:45.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 podman[91262]: 2024-09-13 14:16:45.539768703 +0000 UTC m=+0.505165315 container cleanup c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:45.834 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 bash[91262]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[38255]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[38255]: from='client.26024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[38255]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[29399]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[29399]: from='client.26024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:45 smithi017 ceph-mon[29399]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.105 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 podman[91278]: 2024-09-13 14:16:45.924380159 +0000 UTC m=+0.738871612 container remove c12ac3c783a24231c6410e7da51d86b60e0d8f2d79c23ada17c56460890d60c1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:46.105 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:45 smithi017 podman[91278]: 2024-09-13 14:16:45.983316589 +0000 UTC m=+0.797808004 volume remove 58f022f0222017e75546a49244c932f8ecd14de1cbb422468ef320928fa09d4d 2024-09-13T14:16:46.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:45 smithi189 ceph-mon[33294]: from='client.16110 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:45 smithi189 ceph-mon[33294]: from='client.26024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:46.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:45 smithi189 ceph-mon[33294]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:46.405 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:16:46.405 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:46.405 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.193s CPU time. 2024-09-13T14:16:46.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 systemd[1]: Starting Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:46.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 podman[91435]: 2024-09-13 14:16:46.525243783 +0000 UTC m=+0.017089197 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-13T14:16:46.938 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:47.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:46 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/577111095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:47.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:46 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/577111095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:47.128 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 podman[91435]: 2024-09-13 14:16:46.812136732 +0000 UTC m=+0.303982141 volume create aee832fd16715068611db9b3fe7cf2dc57fd45b16b627440bfc51907b5861e20 2024-09-13T14:16:47.128 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:46 smithi017 podman[91435]: 2024-09-13 14:16:46.920934534 +0000 UTC m=+0.412779937 container create ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:46 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/577111095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:47.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 podman[91435]: 2024-09-13 14:16:47.273767046 +0000 UTC m=+0.765612446 container init ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:47.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 podman[91435]: 2024-09-13 14:16:47.278665663 +0000 UTC m=+0.770511077 container start ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.289Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.289Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.290Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.291Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.314Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.314Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.316Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:47.316Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 bash[91435]: ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 2024-09-13T14:16:47.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:47 smithi017 systemd[1]: Started Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:48.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[38255]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[29399]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:48.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:47 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 systemd[1]: Stopping Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:48.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:47 smithi189 ceph-mon[33294]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:48.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:47 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:47 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:48.530 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 podman[91683]: 2024-09-13 14:16:48.274837438 +0000 UTC m=+0.119904766 container died 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:48.844 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 podman[91683]: 2024-09-13 14:16:48.628631196 +0000 UTC m=+0.473698529 container cleanup 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:48.844 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 bash[91683]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:16:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[29399]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[29399]: from='client.26042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[29399]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:16:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[29399]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:16:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[38255]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[38255]: from='client.26042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[38255]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:16:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:48 smithi017 ceph-mon[38255]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:16:49.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 podman[91700]: 2024-09-13 14:16:48.896963714 +0000 UTC m=+0.625811841 container remove 844957d581ca47c4ac62ce0a99b49f07ca52fb232f280df6603a390ef41cd8c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:49.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:48 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:16:49.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:16:49.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 systemd[1]: Stopped Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:49.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.317s CPU time. 2024-09-13T14:16:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:48 smithi189 ceph-mon[33294]: from='client.16128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:48 smithi189 ceph-mon[33294]: from='client.26042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:48 smithi189 ceph-mon[33294]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:16:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:48 smithi189 ceph-mon[33294]: Deploying daemon node-exporter.a on smithi017 2024-09-13T14:16:49.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:49 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:49.292Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000901518s 2024-09-13T14:16:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 systemd[1]: Starting Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:16:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 bash[91852]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:16:49.846 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:49] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.43.0" 2024-09-13T14:16:49.908 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:49 smithi017 ceph-mon[29399]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:49 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3965378966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:49 smithi017 ceph-mon[38255]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:49 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3965378966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:50.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 bash[91852]: Getting image source signatures 2024-09-13T14:16:50.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 bash[91852]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:16:50.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 bash[91852]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:16:50.201 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:49 smithi017 bash[91852]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:16:50.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:49 smithi189 ceph-mon[33294]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:50.287 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:49 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3965378966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:51.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:51 smithi017 bash[91852]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:16:51.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:51 smithi017 bash[91852]: Writing manifest to image destination 2024-09-13T14:16:52.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[29399]: from='client.26054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[29399]: from='client.16152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[29399]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[38255]: from='client.26054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[38255]: from='client.16152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:51 smithi017 ceph-mon[38255]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.136 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:51 smithi017 podman[91852]: 2024-09-13 14:16:51.784113897 +0000 UTC m=+2.401195829 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:16:52.136 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:51 smithi017 podman[91852]: 2024-09-13 14:16:51.891009647 +0000 UTC m=+2.508091625 container create fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:52.136 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 podman[91852]: 2024-09-13 14:16:52.134815542 +0000 UTC m=+2.751897464 container init fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:51 smithi189 ceph-mon[33294]: from='client.26054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:51 smithi189 ceph-mon[33294]: from='client.16152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:51 smithi189 ceph-mon[33294]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:52.418 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 podman[91852]: 2024-09-13 14:16:52.141502458 +0000 UTC m=+2.758584377 container start fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:16:52.418 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.145Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.145Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.145Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.145Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:16:52.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:16:52.420 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:16:52.420 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:16:52.420 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:16:52.420 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:16:52.420 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:16:52.421 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:16:52.422 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.146Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:16:52.423 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.147Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:16:52.424 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a[91976]: ts=2024-09-13T14:16:52.147Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:16:52.424 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 bash[91852]: fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe 2024-09-13T14:16:52.424 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 14:16:52 smithi017 systemd[1]: Started Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:16:52.894 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/242672925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[29399]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/242672925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:16:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:53 smithi017 ceph-mon[38255]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/242672925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: Deploying daemon prometheus.a on smithi189 2024-09-13T14:16:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:53 smithi189 ceph-mon[33294]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:54 smithi017 ceph-mon[29399]: from='client.26072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:54 smithi017 ceph-mon[29399]: from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:54 smithi017 ceph-mon[38255]: from='client.26072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:54 smithi017 ceph-mon[38255]: from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:54 smithi189 ceph-mon[33294]: from='client.26072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:54 smithi189 ceph-mon[33294]: from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:55 smithi017 ceph-mon[29399]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:55 smithi017 ceph-mon[38255]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:55 smithi189 ceph-mon[33294]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:55.843 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:56 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3893144341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:56 smithi017 ceph-mon[29399]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:56 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3893144341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:56 smithi017 ceph-mon[38255]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:56 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3893144341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:56 smithi189 ceph-mon[33294]: from='client.16182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-mon[29399]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-mon[29399]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-mon[38255]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-mon[38255]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:57.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T14:16:57.294Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00256228s 2024-09-13T14:16:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:57 smithi189 ceph-mon[33294]: from='client.16188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:57 smithi189 ceph-mon[33294]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:58.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:16:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[29649]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:16:58.785 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:16:58.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:58 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/700477591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:58.786 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:58 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/700477591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:58.916 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:58 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/700477591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:16:59.774 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:16:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:16:59] "GET /metrics HTTP/1.1" 200 33996 "" "Prometheus/2.43.0" 2024-09-13T14:16:59.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:59 smithi189 ceph-mon[33294]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:59.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:59 smithi189 ceph-mon[33294]: from='client.26108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:59.775 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:16:59 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[38255]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[38255]: from='client.26108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:16:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[29399]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:16:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[29399]: from='client.26108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:16:59 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:00 smithi017 ceph-mon[29399]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:00 smithi017 ceph-mon[38255]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:00.975 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:00 smithi189 ceph-mon[33294]: from='client.16206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:01.733 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:01 smithi189 ceph-mon[33294]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:01 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/313552563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:01 smithi017 ceph-mon[29399]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:01 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/313552563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:01 smithi017 ceph-mon[38255]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:01 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/313552563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:02.660 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:02 smithi189 ceph-mon[33294]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:02.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:02 smithi017 ceph-mon[29399]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:02.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:02 smithi017 ceph-mon[38255]: from='client.16218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 systemd[1]: Stopping Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.275Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:17:03.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[65789]: ts=2024-09-13T14:17:03.276Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:17:03.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 podman[77508]: 2024-09-13 14:17:03.286010463 +0000 UTC m=+0.104707615 container died acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:03 smithi017 ceph-mon[29399]: from='client.16224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:03 smithi017 ceph-mon[29399]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:03 smithi017 ceph-mon[38255]: from='client.16224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:03 smithi017 ceph-mon[38255]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:04.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-mon[33294]: from='client.16224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:04.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:03 smithi189 ceph-mon[33294]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:04.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 podman[77508]: 2024-09-13 14:17:03.732516838 +0000 UTC m=+0.551213985 container cleanup acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:04.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:03 smithi189 bash[77508]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:17:04.363 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 podman[77525]: 2024-09-13 14:17:04.00889947 +0000 UTC m=+0.723339902 container remove acf8b16b2fc1ebeb7e6c17c430cbb4b38199924115b32af6307c9023ddbd8624 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:04.363 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:17:04.363 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 systemd[1]: Stopped Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:04.363 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.782s CPU time. 2024-09-13T14:17:04.678 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:04.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4065115466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:04.704 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 systemd[1]: Starting Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:04.704 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 podman[77659]: 2024-09-13 14:17:04.473339207 +0000 UTC m=+0.017061184 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:17:04.704 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 podman[77659]: 2024-09-13 14:17:04.594032651 +0000 UTC m=+0.137754618 container create 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:04 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4065115466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:04 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4065115466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 podman[77659]: 2024-09-13 14:17:04.830057859 +0000 UTC m=+0.373779834 container init 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 podman[77659]: 2024-09-13 14:17:04.834508467 +0000 UTC m=+0.378230435 container start 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.854Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.854Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.854Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi189 (none))" 2024-09-13T14:17:05.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.854Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.854Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.856Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.856Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.857Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.857Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.860Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.860Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.682µs 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.860Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.860Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.872Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.872Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.872Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.287µs wal_replay_duration=12.299311ms wbl_replay_duration=135ns total_replay_duration=12.358988ms 2024-09-13T14:17:05.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.875Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.875Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.875Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.891Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.517132ms db_storage=811ns remote_storage=1.24µs web_handler=448ns query_engine=670ns scrape=379.84µs scrape_sd=101.615µs notify=14.204µs notify_sd=176.299µs rules=14.488387ms tracing=5.963µs 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.891Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:04.891Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 bash[77659]: 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a 2024-09-13T14:17:05.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:04 smithi189 systemd[1]: Started Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:05.911 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:05 smithi189 ceph-mon[33294]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:05.911 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:05 smithi189 ceph-mon[33294]: from='client.26144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:05.912 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:05 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:05.912 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:05 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[29399]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[29399]: from='client.26144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[38255]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[38255]: from='client.26144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:05 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:06.682 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:06 smithi189 ceph-mon[33294]: from='client.16242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:06.682 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:06 smithi189 ceph-mon[33294]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:17:06.682 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:06 smithi189 ceph-mon[33294]: Reconfiguring daemon grafana.a on smithi189 2024-09-13T14:17:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[29399]: from='client.16242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[29399]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:17:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[29399]: Reconfiguring daemon grafana.a on smithi189 2024-09-13T14:17:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[38255]: from='client.16242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[38255]: Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:17:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:06 smithi017 ceph-mon[38255]: Reconfiguring daemon grafana.a on smithi189 2024-09-13T14:17:07.626 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:07.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:07 smithi189 ceph-mon[33294]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:07.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:07 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2402537778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:07 smithi017 ceph-mon[29399]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:07 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2402537778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:07 smithi017 ceph-mon[38255]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:07 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2402537778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:08.255 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:07 smithi189 systemd[1]: Stopping Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:08.543 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:08 smithi189 podman[77948]: 2024-09-13 14:17:08.254484559 +0000 UTC m=+0.117651223 container died da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-type=git, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, distribution-scope=public, name=ubi8, release=239.1651231664, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container) 2024-09-13T14:17:08.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:08 smithi189 ceph-mon[33294]: from='client.26162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:08.852 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:08 smithi189 podman[77948]: 2024-09-13 14:17:08.641923548 +0000 UTC m=+0.505090205 container cleanup da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public) 2024-09-13T14:17:08.852 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:08 smithi189 bash[77948]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:17:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:08 smithi017 ceph-mon[29399]: from='client.26162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:08 smithi017 ceph-mon[38255]: from='client.26162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:08 smithi189 podman[77965]: 2024-09-13 14:17:08.851550368 +0000 UTC m=+0.599610902 container remove da25cb53d36f886644224eb1a871ebea899987b4286efdd02a4c13f670470ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, distribution-scope=public, com.redhat.component=ubi8-container, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, release=239.1651231664, io.openshift.tags=base rhel8, io.openshift.expose-services=) 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:08 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 systemd[1]: Stopped Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Consumed 5.421s CPU time. 2024-09-13T14:17:09.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 systemd[1]: Starting Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:09.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 podman[78062]: 2024-09-13 14:17:09.21218858 +0000 UTC m=+0.015090646 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:17:09.384 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 podman[78062]: 2024-09-13 14:17:09.319317001 +0000 UTC m=+0.122219059 container create d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, version=8.5, description=Ceph Grafana Container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.openshift.tags=base rhel8, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git) 2024-09-13T14:17:09.663 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-mon[33294]: from='client.26168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:09.663 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-mon[33294]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:09.664 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 podman[78062]: 2024-09-13 14:17:09.571726442 +0000 UTC m=+0.374628524 container init d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vendor=Red Hat, Inc., description=Ceph Grafana Container, name=ubi8, vcs-type=git, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-13T14:17:09.664 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 podman[78062]: 2024-09-13 14:17:09.576225343 +0000 UTC m=+0.379127403 container start d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, description=Ceph Grafana Container, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, name=ubi8) 2024-09-13T14:17:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:09 smithi017 ceph-mon[29399]: from='client.26168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:09 smithi017 ceph-mon[29399]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:09 smithi017 ceph-mon[38255]: from='client.26168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:09 smithi017 ceph-mon[38255]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 bash[78062]: d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 systemd[1]: Started Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748024838Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748208592Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748227904Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748234992Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:17:09.962 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748241264Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748247639Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748253582Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748259521Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.74826539Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748271402Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748278487Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748284131Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748290817Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=settings t=2024-09-13T14:17:09.748297252Z level=info msg="App mode production" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=sqlstore t=2024-09-13T14:17:09.748349188Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=sqlstore t=2024-09-13T14:17:09.74836657Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=migrator t=2024-09-13T14:17:09.763177633Z level=info msg="Starting DB migrations" 2024-09-13T14:17:09.963 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=migrator t=2024-09-13T14:17:09.767944532Z level=info msg="migrations completed" performed=0 skipped=481 duration=530.947µs 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=sqlstore t=2024-09-13T14:17:09.768421673Z level=info msg="Created default organization" 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=plugin.loader t=2024-09-13T14:17:09.825088192Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=plugin.loader t=2024-09-13T14:17:09.854654035Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=plugin.loader t=2024-09-13T14:17:09.854690206Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=secrets t=2024-09-13T14:17:09.854727604Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=query_data t=2024-09-13T14:17:09.856524041Z level=info msg="Query Service initialization" 2024-09-13T14:17:09.964 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=live.push_http t=2024-09-13T14:17:09.859264043Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:17:10.551 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:10 smithi189 systemd[1]: Stopping Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:10.572 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:10.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-mon[33294]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:17:10.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-mon[33294]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:17:10.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/901157827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:10.803 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=infra.usagestats.collector t=2024-09-13T14:17:10.622162579Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:17:10.803 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=provisioning.datasources t=2024-09-13T14:17:10.711080991Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T14:17:10.803 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=provisioning.datasources t=2024-09-13T14:17:10.711108685Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:17:10.803 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=provisioning.datasources t=2024-09-13T14:17:10.721270164Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:17:10.803 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:10 smithi189 podman[78255]: 2024-09-13 14:17:10.551357348 +0000 UTC m=+0.143750506 container died b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[29399]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[29399]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/901157827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[38255]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:17:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[38255]: Deploying daemon node-exporter.b on smithi189 2024-09-13T14:17:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:10 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/901157827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:11.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=provisioning.alerting t=2024-09-13T14:17:10.802270904Z level=info msg="starting to provision alerting" 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=provisioning.alerting t=2024-09-13T14:17:10.802297296Z level=info msg="finished to provision alerting" 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=grafanaStorageLogger t=2024-09-13T14:17:10.802487313Z level=info msg="storage starting" 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=ngalert.state.manager t=2024-09-13T14:17:10.802607658Z level=info msg="Warming state cache for startup" 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=http.server t=2024-09-13T14:17:10.804918957Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=ngalert.state.manager t=2024-09-13T14:17:10.813764176Z level=info msg="State cache has been initialized" states=0 duration=11.152554ms 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:17:10.813908183Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:17:11.205 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:17:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=ticker t=2024-09-13T14:17:10.813969998Z level=info msg=starting first_tick=2024-09-13T14:17:20Z 2024-09-13T14:17:11.205 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 podman[78255]: 2024-09-13 14:17:11.036109519 +0000 UTC m=+0.628502667 container cleanup b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:11.205 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 bash[78255]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:17:11.469 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 podman[78274]: 2024-09-13 14:17:11.204145955 +0000 UTC m=+0.656709213 container remove b928aa7fd0a110b3aaad2a28d65594a86fd33130a56db9c39e6299fe5d67cc87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:11.469 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:17:11.469 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:17:11.469 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 systemd[1]: Stopped Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:11.469 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.176s CPU time. 2024-09-13T14:17:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:11 smithi189 ceph-mon[33294]: from='client.16272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:11 smithi189 ceph-mon[33294]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:11 smithi189 ceph-mon[33294]: from='client.26186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[29399]: from='client.16272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[29399]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[29399]: from='client.26186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[38255]: from='client.16272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[38255]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:11 smithi017 ceph-mon[38255]: from='client.26186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:12.255 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 systemd[1]: Starting Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:12.255 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:11 smithi189 bash[78409]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:17:12.755 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:12 smithi189 bash[78409]: Getting image source signatures 2024-09-13T14:17:12.756 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:12 smithi189 bash[78409]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:17:12.756 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:12 smithi189 bash[78409]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:17:12.756 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:12 smithi189 bash[78409]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:17:13.514 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:13 smithi017 ceph-mon[29399]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:13 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2781171614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:13 smithi017 ceph-mon[38255]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:13 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2781171614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:13 smithi189 ceph-mon[33294]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:13 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2781171614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:14.721 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:14 smithi189 bash[78409]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:17:14.721 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:14 smithi189 bash[78409]: Writing manifest to image destination 2024-09-13T14:17:14.982 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:14 smithi189 ceph-mon[33294]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:14.983 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:14 smithi189 ceph-mon[33294]: from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:14.983 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:14 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[29399]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[29399]: from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[38255]: from='client.16290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[38255]: from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:14 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:15.253 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:14 smithi189 podman[78409]: 2024-09-13 14:17:14.927926985 +0000 UTC m=+3.069719200 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:17:15.253 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 podman[78409]: 2024-09-13 14:17:15.026455106 +0000 UTC m=+3.168247314 container create 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:15.253 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 podman[78409]: 2024-09-13 14:17:15.25333215 +0000 UTC m=+3.395124372 container init 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:15.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:17:15] "GET /metrics HTTP/1.1" 200 34000 "" "Prometheus/2.51.0" 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 podman[78409]: 2024-09-13 14:17:15.258524294 +0000 UTC m=+3.400316503 container start 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.261Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.261Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.262Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.262Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.262Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.262Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:17:15.506 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:17:15.507 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:17:15.508 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:17:15.509 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:17:15.510 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:17:15.510 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:17:15.510 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b[78485]: ts=2024-09-13T14:17:15.263Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:17:15.510 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 bash[78409]: 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc 2024-09-13T14:17:15.510 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 14:17:15 smithi189 systemd[1]: Started Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:15.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:15.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:15.993 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:15 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:16.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:16.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:16.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:16.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:16.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:16.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:16.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:16.008 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:16.008 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:16.457 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:17.008 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:16 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/882139518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/882139518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:17:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:16 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/882139518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='client.16314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='client.16314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:17 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='client.16308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='client.16314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:18.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:18.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:18.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:18.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:17 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:18 smithi189 ceph-mon[33294]: Upgrade: Updating mgr.y 2024-09-13T14:17:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:18 smithi189 ceph-mon[33294]: Deploying daemon mgr.y on smithi017 2024-09-13T14:17:19.100 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:18 smithi017 systemd[1]: Stopping Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:19.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:18 smithi017 ceph-mon[29399]: Upgrade: Updating mgr.y 2024-09-13T14:17:19.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:18 smithi017 ceph-mon[29399]: Deploying daemon mgr.y on smithi017 2024-09-13T14:17:19.101 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:18 smithi017 ceph-mon[38255]: Upgrade: Updating mgr.y 2024-09-13T14:17:19.101 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:18 smithi017 ceph-mon[38255]: Deploying daemon mgr.y on smithi017 2024-09-13T14:17:19.418 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 podman[93046]: 2024-09-13 14:17:19.0994698 +0000 UTC m=+0.167865904 container stop 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:17:19.418 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 podman[93046]: 2024-09-13 14:17:19.125540485 +0000 UTC m=+0.193936605 container died 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T14:17:19.457 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:19.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 podman[93046]: 2024-09-13 14:17:19.600455254 +0000 UTC m=+0.668851358 container cleanup 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS) 2024-09-13T14:17:19.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 bash[93046]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:17:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:19 smithi189 ceph-mon[33294]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:19 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3075041979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:20.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:19 smithi017 ceph-mon[38255]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:20.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:19 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3075041979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:20.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 podman[93066]: 2024-09-13 14:17:19.827436859 +0000 UTC m=+0.716657175 container remove 566606e62d48d21c929f83379e37dc8f4a11152ca5938aaf6c69e2be97522397 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 systemd[1]: Stopped Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:19 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service: Consumed 47.443s CPU time. 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:19 smithi017 ceph-mon[29399]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:20.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:19 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3075041979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:20.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 systemd[1]: Starting Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:20.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 podman[93247]: 2024-09-13 14:17:20.284095085 +0000 UTC m=+0.016965316 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:17:20.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 podman[93247]: 2024-09-13 14:17:20.403830912 +0000 UTC m=+0.136701178 container create b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T14:17:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:20 smithi017 ceph-mon[29399]: from='client.26234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:20 smithi017 ceph-mon[29399]: from='client.26240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:20 smithi017 ceph-mon[38255]: from='client.26234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:20 smithi017 ceph-mon[38255]: from='client.26240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 podman[93247]: 2024-09-13 14:17:20.66498305 +0000 UTC m=+0.397853274 container init b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:17:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 podman[93247]: 2024-09-13 14:17:20.670274122 +0000 UTC m=+0.403144348 container start b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T14:17:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 ceph-mgr[93285]: -- 172.21.15.17:0/1644156713 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b2ae26b0e0 con 0x55b2ae249000 2024-09-13T14:17:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 bash[93247]: b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 2024-09-13T14:17:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:20 smithi017 systemd[1]: Started Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:20 smithi189 ceph-mon[33294]: from='client.26234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:20 smithi189 ceph-mon[33294]: from='client.26240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:22.178 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:21 smithi189 ceph-mon[33294]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.178 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:21 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.178 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:21 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.178 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:21 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[38255]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[29399]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:21 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:22.411 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:22 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3634881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:22 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3634881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:23.216 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:22 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3634881850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:23 smithi189 ceph-mon[33294]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:23 smithi189 ceph-mon[33294]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:24.149 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:23 smithi189 ceph-mon[33294]: from='client.26264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[38255]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[38255]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:24.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[38255]: from='client.26264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[29399]: from='client.16350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[29399]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:24.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:23 smithi017 ceph-mon[29399]: from='client.26264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:24.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:24 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:25.379 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:25.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:17:25] "GET /metrics HTTP/1.1" 200 34000 "" "Prometheus/2.51.0" 2024-09-13T14:17:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2170274037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:25 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:26.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/2170274037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.063 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2170274037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:26.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:25 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:26.330 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:26] ENGINE Bus STARTING 2024-09-13T14:17:26.330 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: CherryPy Checker: 2024-09-13T14:17:26.330 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: The Application mounted at '' has an empty config. 2024-09-13T14:17:26.330 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:26.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:26] ENGINE Serving on http://:::9283 2024-09-13T14:17:26.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:26] ENGINE Bus STARTED 2024-09-13T14:17:27.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='client.16368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='client.26282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: Standby manager daemon y restarted 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: Standby manager daemon y started 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:17:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:26 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:27.126 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='client.16368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.126 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='client.26282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: Standby manager daemon y restarted 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: Standby manager daemon y started 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='client.16368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='client.26282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: Standby manager daemon y restarted 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: Standby manager daemon y started 2024-09-13T14:17:27.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:17:27.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:27.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:17:27.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:26 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:27 smithi189 ceph-mon[33294]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:27 smithi189 ceph-mon[33294]: mgrmap e27: x(active, since 74s), standbys: y 2024-09-13T14:17:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:27 smithi017 ceph-mon[29399]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:27 smithi017 ceph-mon[29399]: mgrmap e27: x(active, since 74s), standbys: y 2024-09-13T14:17:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:27 smithi017 ceph-mon[38255]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:27 smithi017 ceph-mon[38255]: mgrmap e27: x(active, since 74s), standbys: y 2024-09-13T14:17:28.354 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:28.901 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:28 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/88048442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:28.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:28 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/88048442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:28 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/88048442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:29.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:29] ENGINE Bus STOPPING 2024-09-13T14:17:29.438 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:17:29.095+0000 7fa18bebd640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:17:29.438 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ignoring --setuser ceph since I am not root 2024-09-13T14:17:29.438 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ignoring --setgroup ceph since I am not root 2024-09-13T14:17:29.438 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mgr[68845]: -- 172.21.15.189:0/3561066550 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56168f7b30e0 con 0x56168f791400 2024-09-13T14:17:29.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:17:29.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:29] ENGINE Bus STOPPED 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='client.16386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: Activating manager daemon y 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.25429 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: mgrmap e28: y(active, starting, since 0.00574697s) 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:17:30.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:17:30.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:29 smithi189 ceph-mon[33294]: Manager daemon y is now available 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9567201640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9567201640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9567201640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9567201640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9567201640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9569a06640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9569a06640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9569a06640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9569a06640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.074 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:29.905+0000 7f9569a06640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='client.16386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: Activating manager daemon y 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.25429 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: mgrmap e28: y(active, starting, since 0.00574697s) 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:17:30.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:17:30.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[29399]: Manager daemon y is now available 2024-09-13T14:17:30.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='client.16386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:30.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='client.16392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:30.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 172.21.15.189:0/2983484400' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: Activating manager daemon y 2024-09-13T14:17:30.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.25429 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: mgrmap e28: y(active, starting, since 0.00574697s) 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:17:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:29 smithi017 ceph-mon[38255]: Manager daemon y is now available 2024-09-13T14:17:30.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:30] ENGINE Bus STARTING 2024-09-13T14:17:30.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:30] ENGINE Serving on http://:::9283 2024-09-13T14:17:30.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:17:30] ENGINE Bus STARTED 2024-09-13T14:17:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: mgrmap e29: y(active, since 1.00883s) 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:30 smithi189 ceph-mon[33294]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: mgrmap e29: y(active, since 1.00883s) 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[29399]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:17:31.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:17:31.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: mgrmap e29: y(active, since 1.00883s) 2024-09-13T14:17:31.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:17:31.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:30 smithi017 ceph-mon[38255]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:31.305 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:31.876 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4272351670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:31.876 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: mgrmap e30: y(active, since 2s) 2024-09-13T14:17:31.876 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: [13/Sep/2024:14:17:31] ENGINE Bus STARTING 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: [13/Sep/2024:14:17:31] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: [13/Sep/2024:14:17:31] ENGINE Error in HTTPServer.serve 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:17:31.877 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: [13/Sep/2024:14:17:31] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: [13/Sep/2024:14:17:31] ENGINE Bus STARTED 2024-09-13T14:17:31.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[29399]: from='client.26351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4272351670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: mgrmap e30: y(active, since 2s) 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: [13/Sep/2024:14:17:31] ENGINE Bus STARTING 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: [13/Sep/2024:14:17:31] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: [13/Sep/2024:14:17:31] ENGINE Error in HTTPServer.serve 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:17:31.879 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:17:31.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: [13/Sep/2024:14:17:31] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: [13/Sep/2024:14:17:31] ENGINE Bus STARTED 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-mon[38255]: from='client.26351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:31.481+0000 7f954d793640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:17:31] ENGINE Error in HTTPServer.serve 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: Traceback (most recent call last): 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._connections.run(self.expiration_interval) 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._run(expiration_interval) 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:17:31.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: s = self.context.wrap_socket( 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: return self.sslsocket_class._create( 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self.do_handshake() 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._sslobj.do_handshake() 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:17:31.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:17:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4272351670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: mgrmap e30: y(active, since 2s) 2024-09-13T14:17:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: [13/Sep/2024:14:17:31] ENGINE Bus STARTING 2024-09-13T14:17:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: [13/Sep/2024:14:17:31] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: [13/Sep/2024:14:17:31] ENGINE Error in HTTPServer.serve 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: Traceback (most recent call last): 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._run(expiration_interval) 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: s = self.context.wrap_socket( 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: return self.sslsocket_class._create( 2024-09-13T14:17:32.006 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: self.do_handshake() 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._sslobj.do_handshake() 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: [13/Sep/2024:14:17:31] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: [13/Sep/2024:14:17:31] ENGINE Bus STARTED 2024-09-13T14:17:32.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:31 smithi189 ceph-mon[33294]: from='client.26351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:34.275 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:34.275 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[29399]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.275 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[29399]: mgrmap e31: y(active, since 4s) 2024-09-13T14:17:34.275 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/146975639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:34.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[38255]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.276 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[38255]: mgrmap e31: y(active, since 4s) 2024-09-13T14:17:34.276 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:34 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/146975639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:34.423 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:34.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-mon[33294]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:34.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-mon[33294]: mgrmap e31: y(active, since 4s) 2024-09-13T14:17:34.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/146975639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:34.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:17:34] ENGINE Bus STARTING 2024-09-13T14:17:34.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: CherryPy Checker: 2024-09-13T14:17:34.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: The Application mounted at '' has an empty config. 2024-09-13T14:17:34.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: 2024-09-13T14:17:34.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:17:34] ENGINE Serving on http://:::9283 2024-09-13T14:17:34.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: [13/Sep/2024:14:17:34] ENGINE Bus STARTED 2024-09-13T14:17:35.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='client.26369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='client.16470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='client.26369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: Standby manager daemon x started 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='client.16470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:35 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[68822]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:17:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-13T14:17:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='client.26369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1677228214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='client.16470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:35 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[29399]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[29399]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[38255]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:17:36.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[38255]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-13T14:17:36.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:36 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-mon[33294]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:17:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-mon[33294]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-13T14:17:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:37.238 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:37 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3193908639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:37.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:37.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:37 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3193908639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:37.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:37 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3193908639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:17:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='client.26387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='client.16488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='client.26387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='client.16488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:38 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:17:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='client.26387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='client.16488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:38.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:38.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:38 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:39 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:17:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:17:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:17:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:17:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:39.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:39 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:40.185 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:40.336 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:40.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:17:40.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:40.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:17:40.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:40 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/4275496104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:17:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/4275496104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: Reconfiguring daemon prometheus.a on smithi189 2024-09-13T14:17:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:40 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/4275496104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:41.350 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:41 smithi189 ceph-mon[33294]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:41.350 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:41 smithi189 ceph-mon[33294]: from='client.26411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:41 smithi017 ceph-mon[29399]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:41 smithi017 ceph-mon[29399]: from='client.26411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:41 smithi017 ceph-mon[38255]: from='client.26405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:41 smithi017 ceph-mon[38255]: from='client.26411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:42 smithi017 ceph-mon[29399]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:17:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:42 smithi017 ceph-mon[38255]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:17:42.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-mon[33294]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:17:42.755 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 systemd[1]: Stopping Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:43.129 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:17:43.137 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.854Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.856Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.856Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[77674]: ts=2024-09-13T14:17:42.856Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T14:17:43.138 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:42 smithi189 podman[81105]: 2024-09-13 14:17:42.86507755 +0000 UTC m=+0.128993521 container died 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:43 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/579501976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:43 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/579501976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:43.478 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:43 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/579501976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:43.478 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 podman[81105]: 2024-09-13 14:17:43.235400529 +0000 UTC m=+0.499316499 container cleanup 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:43.478 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 bash[81105]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:17:43.755 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 podman[81122]: 2024-09-13 14:17:43.478125649 +0000 UTC m=+0.614228600 container remove 4ba00b5181919d5338e84d15768fe74bb601ca04da256454ac71007c81d2b59a (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:43.755 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:17:43.755 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 systemd[1]: Stopped Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:43.755 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.124s CPU time. 2024-09-13T14:17:43.756 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 systemd[1]: Starting Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:44.064 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 podman[81218]: 2024-09-13 14:17:43.711926116 +0000 UTC m=+0.016820634 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:17:44.065 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:43 smithi189 podman[81218]: 2024-09-13 14:17:43.85439117 +0000 UTC m=+0.159285714 container create 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[29399]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[29399]: from='client.26423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[29399]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[38255]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[38255]: from='client.26423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:44 smithi017 ceph-mon[38255]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-mon[33294]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-mon[33294]: from='client.26423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-mon[33294]: from='client.26429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 podman[81218]: 2024-09-13 14:17:44.064705906 +0000 UTC m=+0.369600407 container init 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 podman[81218]: 2024-09-13 14:17:44.069776238 +0000 UTC m=+0.374670740 container start 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:17:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.089Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:17:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.089Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:17:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.089Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi189 (none))" 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.089Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.089Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.092Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.092Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.094Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.094Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.095Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.095Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.271µs 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.095Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:17:44.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.096Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.107Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.111Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.111Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.111Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.844µs wal_replay_duration=16.078991ms wbl_replay_duration=173ns total_replay_duration=16.142292ms 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.116Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.116Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.116Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.130Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.852858ms db_storage=888ns remote_storage=1.025µs web_handler=337ns query_engine=1.018µs scrape=460.322µs scrape_sd=93.733µs notify=11.185µs notify_sd=13.137µs rules=12.928406ms tracing=5.271µs 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.130Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:44.130Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:17:44.508 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 bash[81218]: 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b 2024-09-13T14:17:44.509 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:44 smithi189 systemd[1]: Started Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:45 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:17:45.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:45.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:45 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:17:46.082 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: Upgrade: Updating mgr.x 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:46.250 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.251 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: Deploying daemon mgr.x on smithi189 2024-09-13T14:17:46.251 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3513330794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: Upgrade: Updating mgr.x 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: Deploying daemon mgr.x on smithi189 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3513330794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: Upgrade: Updating mgr.x 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: Deploying daemon mgr.x on smithi189 2024-09-13T14:17:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:46 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3513330794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:47.204 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:47 smithi189 ceph-mon[33294]: from='client.16536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:47 smithi189 ceph-mon[33294]: from='client.16542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:47 smithi189 systemd[1]: Stopping Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:47 smithi017 ceph-mon[29399]: from='client.16536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:47 smithi017 ceph-mon[29399]: from='client.16542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:47 smithi017 ceph-mon[38255]: from='client.16536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:47 smithi017 ceph-mon[38255]: from='client.16542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:47.885 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:47 smithi189 podman[81623]: 2024-09-13 14:17:47.624932208 +0000 UTC m=+0.147163584 container died ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T14:17:48.205 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:47 smithi189 podman[81623]: 2024-09-13 14:17:47.974654393 +0000 UTC m=+0.496885766 container cleanup ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T14:17:48.205 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:47 smithi189 bash[81623]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:17:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:48 smithi189 ceph-mon[33294]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:48.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 podman[81642]: 2024-09-13 14:17:48.318125921 +0000 UTC m=+0.695965486 container remove ba244c09f7be16fef19564346f59b4d48ab88a58af532cb1b13d1bfb206126bb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:17:48.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:17:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:48 smithi017 ceph-mon[29399]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:48 smithi017 ceph-mon[38255]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:17:48.971 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:17:48.971 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 systemd[1]: Stopped Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:48.971 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service: Consumed 18.403s CPU time. 2024-09-13T14:17:48.971 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 systemd[1]: Starting Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:17:49.036 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:49.233 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:49 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/589236780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:49.233 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:48 smithi189 podman[81776]: 2024-09-13 14:17:48.873437577 +0000 UTC m=+0.015786290 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:17:49.233 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 podman[81776]: 2024-09-13 14:17:49.005059166 +0000 UTC m=+0.147407865 container create e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:17:49.234 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 podman[81776]: 2024-09-13 14:17:49.232933096 +0000 UTC m=+0.375281798 container init e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:17:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:49 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/589236780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:49 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/589236780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:49.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 podman[81776]: 2024-09-13 14:17:49.238072038 +0000 UTC m=+0.380420741 container start e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:17:49.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 ceph-mgr[81813]: -- 172.21.15.189:0/2155633817 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d302ab10e0 con 0x55d302a8e800 2024-09-13T14:17:49.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 bash[81776]: e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 2024-09-13T14:17:49.505 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:49 smithi189 systemd[1]: Started Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:17:50.404 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:50 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='client.26459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='client.26465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:50.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:50.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:50 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:51.995 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:52.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:52 smithi189 ceph-mon[33294]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:52.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:52 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/909577669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:52 smithi017 ceph-mon[29399]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:52 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/909577669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:52 smithi017 ceph-mon[38255]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:52 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/909577669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:53.678 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:53 smithi189 ceph-mon[33294]: from='client.16578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:53.679 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:53 smithi189 ceph-mon[33294]: from='client.16584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:53 smithi017 ceph-mon[38255]: from='client.16578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:53 smithi017 ceph-mon[38255]: from='client.16584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:53 smithi017 ceph-mon[29399]: from='client.16578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:53 smithi017 ceph-mon[29399]: from='client.16584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[38255]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[29399]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:54 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-mon[33294]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:17:54] ENGINE Bus STARTING 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: CherryPy Checker: 2024-09-13T14:17:54.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: The Application mounted at '' has an empty config. 2024-09-13T14:17:54.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: 2024-09-13T14:17:54.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:17:54] ENGINE Serving on http://:::9283 2024-09-13T14:17:54.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:17:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:17:54] ENGINE Bus STARTED 2024-09-13T14:17:54.940 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: Standby manager daemon x restarted 2024-09-13T14:17:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: Standby manager daemon x started 2024-09-13T14:17:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1335116175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: Standby manager daemon x restarted 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:17:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:55 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1335116175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: Standby manager daemon x restarted 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/1210817808' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:17:55.729 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.730 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:55.730 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:55 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1335116175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:56.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[29399]: from='client.26501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[29399]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:56.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[29399]: from='client.26507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[29399]: mgrmap e33: y(active, since 26s), standbys: x 2024-09-13T14:17:56.420 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[38255]: from='client.26501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[38255]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:56.611 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:56 smithi189 ceph-mon[33294]: from='client.26501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.612 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:56 smithi189 ceph-mon[33294]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:56.612 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:56 smithi189 ceph-mon[33294]: from='client.26507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.612 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:56 smithi189 ceph-mon[33294]: mgrmap e33: y(active, since 26s), standbys: x 2024-09-13T14:17:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[38255]: from='client.26507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:56 smithi017 ceph-mon[38255]: mgrmap e33: y(active, since 26s), standbys: x 2024-09-13T14:17:57.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:17:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:17:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:17:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:17:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:17:57] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.51.0" 2024-09-13T14:17:57.893 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:17:58.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:58.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/3723523484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:58.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3723523484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:58.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:17:58.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:58 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:17:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:17:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3723523484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:17:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:17:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:58 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:17:59.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:59 smithi189 ceph-mon[33294]: from='client.26519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:59.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:59 smithi189 ceph-mon[33294]: Upgrade: Setting container_image for all mgr 2024-09-13T14:17:59.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:59 smithi189 ceph-mon[33294]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:17:59.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:17:59 smithi189 ceph-mon[33294]: from='client.26525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[29399]: from='client.26519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[29399]: Upgrade: Setting container_image for all mgr 2024-09-13T14:17:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[29399]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:17:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[29399]: from='client.26525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[38255]: from='client.26519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[38255]: Upgrade: Setting container_image for all mgr 2024-09-13T14:17:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[38255]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:17:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:17:59 smithi017 ceph-mon[38255]: from='client.26525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: Upgrade: Updating mon.a 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: Deploying daemon mon.a on smithi017 2024-09-13T14:18:00.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-mon[33294]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:00.495 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: Upgrade: Updating mon.a 2024-09-13T14:18:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:18:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: Deploying daemon mon.a on smithi017 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[38255]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: Upgrade: Updating mon.a 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: from='mgr.26246 ' entity='mgr.y' 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:00.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: Deploying daemon mon.a on smithi017 2024-09-13T14:18:00.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:00 smithi017 ceph-mon[29399]: from='mgr.26246 172.21.15.17:0/2763848541' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:18:00.850 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:18:01.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:01 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/1438232357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:01.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:01 smithi017 systemd[1]: Stopping Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:18:01.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:01 smithi017 ceph-mon[29399]: from='client.? 172.21.15.17:0/1438232357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:01 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/1438232357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:01.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:01 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[29376]: 2024-09-13T14:18:01.529+0000 7f1725797640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:18:01.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:01 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[29376]: 2024-09-13T14:18:01.529+0000 7f1725797640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T14:18:01.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:01 smithi017 podman[102810]: 2024-09-13 14:18:01.560613042 +0000 UTC m=+0.166943999 container died 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T14:18:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 podman[102810]: 2024-09-13 14:18:02.002259214 +0000 UTC m=+0.608590203 container cleanup 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T14:18:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 bash[102810]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:18:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 podman[102827]: 2024-09-13 14:18:02.254174422 +0000 UTC m=+0.693065987 container remove 7f6206d3dbf8b30fb6048696c4d82a77023e6a06330ef3c0e2fe11eb57e58826 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T14:18:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:18:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 systemd[1]: Stopped Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:18:02.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service: Consumed 13.968s CPU time. 2024-09-13T14:18:02.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 systemd[1]: Starting Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:18:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 podman[102962]: 2024-09-13 14:18:02.824105106 +0000 UTC m=+0.016540346 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:18:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:02 smithi017 podman[102962]: 2024-09-13 14:18:02.947127712 +0000 UTC m=+0.139562935 container create c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T14:18:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 podman[102962]: 2024-09-13 14:18:03.200285947 +0000 UTC m=+0.392721193 container init c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 podman[102962]: 2024-09-13 14:18:03.205810591 +0000 UTC m=+0.398245814 container start c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_CLEAN=True, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: pidfile_write: ignore empty --pid-file 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: load: jerasure load: lrc 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Git sha 0 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: DB SUMMARY 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: DB Session ID: 2ZCUYV4V66ANIKVS1TU8 2024-09-13T14:18:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1172 Bytes 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 55206 ; 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.env: 0x55f189a9f160 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:18:03.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.info_log: 0x55f18b311880 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.statistics: (nil) 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.use_fsync: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:18:03.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.db_log_dir: 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.wal_dir: 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.write_buffer_manager: 0x55f18b314960 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:18:03.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.unordered_write: 0 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.row_cache: None 2024-09-13T14:18:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.wal_filter: None 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.wal_compression: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:18:03.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:18:03.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_open_files: -1 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Compression algorithms supported: 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kZSTD supported: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:18:03.709 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.merge_operator: 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_filter: None 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:18:03.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f18b3114e0) 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 4 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x55f18b304850 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:18:03.711 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2024-09-13T14:18:03.712 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: bloomfilter 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 5 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_auto_readahead_size: 262144 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: prepopulate_block_cache: 0 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:18:03.713 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression: NoCompression 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.num_levels: 7 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:18:03.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:18:03.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:18:03.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:18:03.717 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:18:03.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.ttl: 2592000 2024-09-13T14:18:03.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:18:03.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12300, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 48c2b631-e091-4dda-a6ae-b1bde03a85b8 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237083235361, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237083236391, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 55697, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12301, "largest_seqno": 12380, "table_properties": {"data_size": 54232, "index_size": 235, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 197, "raw_key_size": 1489, "raw_average_key_size": 26, "raw_value_size": 53011, "raw_average_value_size": 930, "num_data_blocks": 10, "num_entries": 57, "num_filter_entries": 57, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237083, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "48c2b631-e091-4dda-a6ae-b1bde03a85b8", "db_session_id": "2ZCUYV4V66ANIKVS1TU8", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237083236543, "job": 1, "event": "recovery_finished"} 2024-09-13T14:18:03.721 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f18b334e00 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: DB pointer 0x55f18b434000 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:18:03.722 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 1/0 54.39 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 80.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: L6 1/0 7.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 7.51 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 80.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 80.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 80.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.723 INFO:journalctl@ceph.mon.a.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 11.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 11.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache BinnedLRUCache@0x55f18b304850#7 capacity: 512.00 MB usage: 0.55 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.23 KB,4.47035e-05%) IndexBlock(1,0.31 KB,5.96046e-05%) Misc(1,0.00 KB,0%) 2024-09-13T14:18:03.724 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???) e3 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).mds e9 new map 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).mds e9 print_map 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: e9 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: 1 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: Filesystem 'foo' (1) 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: fs_name foo 2024-09-13T14:18:03.725 INFO:journalctl@ceph.mon.a.smithi017.stdout: epoch 8 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: created 2024-09-13T14:13:41.125713+0000 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: modified 2024-09-13T14:14:00.359832+0000 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: tableserver 0 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: root 0 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_timeout 60 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_autoclose 300 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_file_size 1099511627776 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_xattr_size 65536 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: required_client_features {} 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure 0 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure_osd_epoch 0 2024-09-13T14:18:03.726 INFO:journalctl@ceph.mon.a.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_mds 1 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: in 0 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: up {0=24539} 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: failed 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: damaged 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: stopped 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_pools [4] 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_pool 3 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: inline_data disabled 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: balancer 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: bal_rank_mask -1 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: standby_count_wanted 1 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:18:03.727 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi017.ldrmqj{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout: Standby daemons: 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi189.ndtccr{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).osd e81 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 ceph-mon[103001]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 bash[102962]: c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 2024-09-13T14:18:03.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:03 smithi017 systemd[1]: Started Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:18:03.803 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:18:04.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:06.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:06.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:06.492 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:18:06.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ignoring --setuser ceph since I am not root 2024-09-13T14:18:06.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ignoring --setgroup ceph since I am not root 2024-09-13T14:18:06.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mgr[93285]: -- 172.21.15.17:0/935404851 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55951d9f50e0 con 0x55951d9d2800 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='client.16632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='client.26543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/747794294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='client.26555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='client.26561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: mon.a calling monitor election 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:18:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: monmap epoch 3 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: min_mon_release 17 (quincy) 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: election_strategy: 1 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: from='mgr.26246 ' entity='' 2024-09-13T14:18:06.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[103001]: mgrmap e34: y(active, since 37s), standbys: x 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='client.16632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='client.26543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/747794294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='client.26555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='client.26561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: mon.a calling monitor election 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: monmap epoch 3 2024-09-13T14:18:06.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: min_mon_release 17 (quincy) 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: election_strategy: 1 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: overall HEALTH_OK 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: from='mgr.26246 ' entity='' 2024-09-13T14:18:06.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:06 smithi017 ceph-mon[38255]: mgrmap e34: y(active, since 37s), standbys: x 2024-09-13T14:18:06.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: ignoring --setuser ceph since I am not root 2024-09-13T14:18:06.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: ignoring --setgroup ceph since I am not root 2024-09-13T14:18:06.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mgr[81813]: -- 172.21.15.189:0/484833730 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5603b30b50e0 con 0x5603b3092800 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='client.16632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='client.26543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/747794294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='client.26555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='client.26561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: mon.a calling monitor election 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:18:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: monmap epoch 3 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: min_mon_release 17 (quincy) 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: election_strategy: 1 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: mgrmap e33: y(active, since 37s), standbys: x 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: overall HEALTH_OK 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: from='mgr.26246 ' entity='' 2024-09-13T14:18:06.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-mon[33294]: mgrmap e34: y(active, since 37s), standbys: x 2024-09-13T14:18:07.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:18:07.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2691960976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:07 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2691960976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:07.475 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:18:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:07 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2691960976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:18:10.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: Standby manager daemon x restarted 2024-09-13T14:18:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: Standby manager daemon x started 2024-09-13T14:18:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:18:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[103001]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: Standby manager daemon x restarted 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: Standby manager daemon x started 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:18:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:11 smithi017 ceph-mon[38255]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:18:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: Standby manager daemon x restarted 2024-09-13T14:18:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: Standby manager daemon x started 2024-09-13T14:18:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:18:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-mon[33294]: from='mgr.? 172.21.15.189:0/2125242462' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:18:11] ENGINE Bus STARTING 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: CherryPy Checker: 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: The Application mounted at '' has an empty config. 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:18:11] ENGINE Serving on http://:::9283 2024-09-13T14:18:11.756 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 14:18:11 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x[81790]: [13/Sep/2024:14:18:11] ENGINE Bus STARTED 2024-09-13T14:18:12.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.111+0000 7f90b2576640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.111+0000 7f90b2576640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.111+0000 7f90b2576640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.111+0000 7f90b2576640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.111+0000 7f90b2576640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.112+0000 7f90b557c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.112+0000 7f90b557c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.112+0000 7f90b557c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.112+0000 7f90b557c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.112+0000 7f90b557c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: mgrmap e35: y(active, since 42s), standbys: x 2024-09-13T14:18:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: Active manager daemon y restarted 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: Activating manager daemon y 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: mgrmap e36: y(active, starting, since 0.00760134s), standbys: x 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:18:12.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: Manager daemon y is now available 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:12.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: mgrmap e35: y(active, since 42s), standbys: x 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: Active manager daemon y restarted 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: Activating manager daemon y 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: mgrmap e36: y(active, starting, since 0.00760134s), standbys: x 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:18:12.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:18:12.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: Manager daemon y is now available 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:18:12.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:18:12.643 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (86s) 38s ago 5m 18.8M - 0.27.0 11f11916f8cd ceb612349939 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (63s) 18s ago 5m 81.9M - 9.4.7 954c08fa6188 d0c012ef79a6 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (4m) 38s ago 4m 16.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (4m) 18s ago 4m 13.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443,9283,8765 running (23s) 18s ago 9m 71.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 e2be48bfd3a8 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (52s) 38s ago 11m 489M - 19.3.0-4697-gd58f7306 fec07016a2b0 b59080328aea 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (11m) 38s ago 11m 56.2M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 7f6206d3dbf8 2024-09-13T14:18:12.644 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (9m) 18s ago 9m 45.5M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 6439c9112cda 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (9m) 38s ago 9m 46.3M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 67f2b6c9b084 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (80s) 38s ago 5m 9671k - 1.7.0 72c9c2088986 fce992870ef4 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (57s) 18s ago 5m 9.90M - 1.7.0 72c9c2088986 68ff3e2b1910 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (9m) 38s ago 9m 47.5M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 61ff807d7240 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (8m) 38s ago 8m 52.5M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b8f0ae12aacb 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (8m) 38s ago 8m 45.9M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2e705cedcfaf 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (7m) 38s ago 7m 43.1M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 81c4e9274ff0 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (7m) 18s ago 7m 47.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (7m) 18s ago 7m 46.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (6m) 18s ago 6m 45.3M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (6m) 18s ago 6m 52.5M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:18:12.645 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (28s) 18s ago 5m 39.4M - 2.51.0 1d3b7f56885b 770a1c26888c 2024-09-13T14:18:12.745 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.630+0000 7f90e4868640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:18:12] ENGINE Bus STARTING 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: CherryPy Checker: 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: The Application mounted at '' has an empty config. 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:18:12] ENGINE Serving on http://:::9283 2024-09-13T14:18:12.746 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:12 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:14:18:12] ENGINE Bus STARTED 2024-09-13T14:18:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: mgrmap e35: y(active, since 42s), standbys: x 2024-09-13T14:18:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: Active manager daemon y restarted 2024-09-13T14:18:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: Activating manager daemon y 2024-09-13T14:18:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:18:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: mgrmap e36: y(active, starting, since 0.00760134s), standbys: x 2024-09-13T14:18:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi189.ndtccr"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.ldrmqj"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:18:12.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:18:12.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:18:12.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:18:12.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:18:12.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:18:12.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:18:12.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:18:12.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:18:12.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:18:12.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:18:12.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:18:12.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:18:12.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: Manager daemon y is now available 2024-09-13T14:18:12.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:12.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:12.760 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:18:12.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:18:12.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:18:13.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:13.370+0000 7f9098348640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:18:13] ENGINE Error in HTTPServer.serve 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: Traceback (most recent call last): 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._connections.run(self.expiration_interval) 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:18:13.626 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._run(expiration_interval) 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: s = self.context.wrap_socket( 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: return self.sslsocket_class._create( 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self.do_handshake() 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: self._sslobj.do_handshake() 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:18:13.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:18:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:13 smithi189 ceph-mon[33294]: mgrmap e37: y(active, since 1.01032s), standbys: x 2024-09-13T14:18:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:13 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:13 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:13 smithi189 ceph-mon[33294]: [13/Sep/2024:14:18:13] ENGINE Bus STARTING 2024-09-13T14:18:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[103001]: mgrmap e37: y(active, since 1.01032s), standbys: x 2024-09-13T14:18:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[103001]: [13/Sep/2024:14:18:13] ENGINE Bus STARTING 2024-09-13T14:18:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[38255]: mgrmap e37: y(active, since 1.01032s), standbys: x 2024-09-13T14:18:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:13 smithi017 ceph-mon[38255]: [13/Sep/2024:14:18:13] ENGINE Bus STARTING 2024-09-13T14:18:14.306 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:18:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: [13/Sep/2024:14:18:13] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:18:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: [13/Sep/2024:14:18:13] ENGINE Error in HTTPServer.serve 2024-09-13T14:18:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:18:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:18:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:18:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: [13/Sep/2024:14:18:13] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: [13/Sep/2024:14:18:13] ENGINE Bus STARTED 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[103001]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: [13/Sep/2024:14:18:13] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: [13/Sep/2024:14:18:13] ENGINE Error in HTTPServer.serve 2024-09-13T14:18:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:18:14.953 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: [13/Sep/2024:14:18:13] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: [13/Sep/2024:14:18:13] ENGINE Bus STARTED 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:14.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:14 smithi017 ceph-mon[38255]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-13T14:18:15.151 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: [13/Sep/2024:14:18:13] ENGINE Serving on https://172.21.15.17:7150 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: [13/Sep/2024:14:18:13] ENGINE Error in HTTPServer.serve 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: Traceback (most recent call last): 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._run(expiration_interval) 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:18:15.152 INFO:journalctl@ceph.mon.b.smithi189.stdout: s = self.context.wrap_socket( 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: return self.sslsocket_class._create( 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: self.do_handshake() 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: self._sslobj.do_handshake() 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: [13/Sep/2024:14:18:13] ENGINE Serving on http://172.21.15.17:8765 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: [13/Sep/2024:14:18:13] ENGINE Bus STARTED 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:15.153 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:14 smithi189 ceph-mon[33294]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-13T14:18:16.764 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:18:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[103001]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[103001]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-13T14:18:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[38255]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[38255]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-13T14:18:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:16 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:16 smithi189 ceph-mon[33294]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:16 smithi189 ceph-mon[33294]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-13T14:18:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:16 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:17.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:16 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:18:17] "GET /metrics HTTP/1.1" 200 31300 "" "Prometheus/2.51.0" 2024-09-13T14:18:18.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: from='client.26621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:18 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: from='client.26621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: from='client.26621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:18:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:19.045 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:18:19.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:19 smithi189 ceph-mon[33294]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:19.300 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:19 smithi017 ceph-mon[38255]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:19.300 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:19 smithi017 ceph-mon[103001]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:20.447 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:20.448 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:20 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:20.448 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:20 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:20.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:20.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:20.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:20 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:20.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:20 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:21 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:18:22.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:22.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-13T14:18:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.conf 2024-09-13T14:18:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:18:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:18:23.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[38255]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[38255]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[38255]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:18:23.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[103001]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[103001]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:23 smithi017 ceph-mon[103001]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:18:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:23 smithi189 ceph-mon[33294]: Updating smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:23 smithi189 ceph-mon[33294]: Updating smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:18:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:23 smithi189 ceph-mon[33294]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:18:24.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:24 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:25 smithi017 ceph-mon[103001]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:18:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:25 smithi017 ceph-mon[38255]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:18:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:25 smithi189 ceph-mon[33294]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:18:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:27.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:26 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:27 smithi017 ceph-mon[103001]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:18:27] "GET /metrics HTTP/1.1" 200 33999 "" "Prometheus/2.51.0" 2024-09-13T14:18:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:27 smithi017 ceph-mon[38255]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:27 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:27 smithi189 ceph-mon[33294]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:27 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:29 smithi017 ceph-mon[103001]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:29 smithi017 ceph-mon[38255]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:29 smithi189 ceph-mon[33294]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:31 smithi017 ceph-mon[103001]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:31 smithi017 ceph-mon[38255]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:31 smithi189 ceph-mon[33294]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:18:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:33 smithi017 ceph-mon[103001]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:33 smithi017 ceph-mon[38255]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:33 smithi189 ceph-mon[33294]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:34.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:35 smithi017 ceph-mon[103001]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:35 smithi017 ceph-mon[38255]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:35.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:35 smithi189 ceph-mon[33294]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:37.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:36.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:37 smithi017 ceph-mon[103001]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:37 smithi017 ceph-mon[38255]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:37.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:18:37] "GET /metrics HTTP/1.1" 200 33999 "" "Prometheus/2.51.0" 2024-09-13T14:18:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:37 smithi189 ceph-mon[33294]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:39 smithi017 ceph-mon[103001]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:39 smithi017 ceph-mon[38255]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:39 smithi189 ceph-mon[33294]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:40 smithi189 ceph-mon[33294]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:40 smithi017 ceph-mon[103001]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:40 smithi017 ceph-mon[38255]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:42 smithi017 ceph-mon[103001]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:42 smithi017 ceph-mon[38255]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:42 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:42 smithi189 ceph-mon[33294]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:42 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:44.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:44 smithi017 ceph-mon[103001]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:44 smithi017 ceph-mon[38255]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:44 smithi189 ceph-mon[33294]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:46.949 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:46 smithi189 ceph-mon[33294]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:46 smithi017 ceph-mon[38255]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:46 smithi017 ceph-mon[103001]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:47.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:18:47] "GET /metrics HTTP/1.1" 200 33994 "" "Prometheus/2.51.0" 2024-09-13T14:18:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:48 smithi017 ceph-mon[103001]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:48 smithi017 ceph-mon[38255]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:48 smithi189 ceph-mon[33294]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:50 smithi017 ceph-mon[103001]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:50 smithi017 ceph-mon[38255]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:50 smithi189 ceph-mon[33294]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:52 smithi189 ceph-mon[33294]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:52 smithi017 ceph-mon[103001]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:52 smithi017 ceph-mon[38255]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:54.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:54 smithi189 ceph-mon[33294]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:54 smithi017 ceph-mon[103001]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:54 smithi017 ceph-mon[38255]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:56 smithi189 ceph-mon[33294]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:18:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:18:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:18:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:56 smithi017 ceph-mon[103001]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:56 smithi017 ceph-mon[38255]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:18:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:18:57] "GET /metrics HTTP/1.1" 200 33995 "" "Prometheus/2.51.0" 2024-09-13T14:18:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:57 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:58.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:57 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:18:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:18:59 smithi017 ceph-mon[103001]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:18:59 smithi017 ceph-mon[38255]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:18:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:18:59 smithi189 ceph-mon[33294]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:01 smithi017 ceph-mon[103001]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:01 smithi017 ceph-mon[38255]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:01 smithi189 ceph-mon[33294]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:03 smithi017 ceph-mon[103001]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:03 smithi017 ceph-mon[38255]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:03 smithi189 ceph-mon[33294]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:04.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:05 smithi017 ceph-mon[103001]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:05 smithi017 ceph-mon[38255]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:05.740 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:05 smithi189 ceph-mon[33294]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:07.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:07 smithi017 ceph-mon[103001]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:07 smithi017 ceph-mon[38255]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:07.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:07] "GET /metrics HTTP/1.1" 200 33995 "" "Prometheus/2.51.0" 2024-09-13T14:19:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:07 smithi189 ceph-mon[33294]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:09 smithi017 ceph-mon[103001]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:09 smithi017 ceph-mon[38255]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:09 smithi189 ceph-mon[33294]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:10.493 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:11 smithi189 ceph-mon[33294]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:11 smithi017 ceph-mon[103001]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:11 smithi017 ceph-mon[38255]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:12 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:12 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:13 smithi017 ceph-mon[103001]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:13 smithi017 ceph-mon[38255]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:13 smithi189 ceph-mon[33294]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:14.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:15 smithi017 ceph-mon[103001]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:15 smithi017 ceph-mon[38255]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:15.740 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:15 smithi189 ceph-mon[33294]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:16.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:16.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:17.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:16 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:17 smithi017 ceph-mon[103001]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:17 smithi017 ceph-mon[38255]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:17] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.51.0" 2024-09-13T14:19:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:17 smithi189 ceph-mon[33294]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:19 smithi017 ceph-mon[103001]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:19 smithi017 ceph-mon[38255]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:19 smithi189 ceph-mon[33294]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:21.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:21 smithi017 ceph-mon[103001]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:21.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:21 smithi017 ceph-mon[38255]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:21.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:19:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:21 smithi189 ceph-mon[33294]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:21.887 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:22.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:22 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:22 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:23.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:23 smithi017 ceph-mon[103001]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:23.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:23 smithi017 ceph-mon[38255]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:23 smithi189 ceph-mon[33294]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:23.989 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:24.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:24 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:24.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-13T14:19:25.157 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[103001]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[103001]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[103001]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[38255]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[38255]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[38255]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:24 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:24 smithi189 ceph-mon[33294]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:24 smithi189 ceph-mon[33294]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:24 smithi189 ceph-mon[33294]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:24 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:27.202 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:26 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:27.486 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:27] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.51.0" 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[103001]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[38255]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.487 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:27 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:27 smithi189 ceph-mon[33294]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:27 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:27 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:27 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:19:27.595 INFO:teuthology.orchestra.run.smithi017.stdout:wait for majority of mons to upgrade 2024-09-13T14:19:28.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:19:29.109 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:29 smithi189 ceph-mon[33294]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:29.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:29 smithi017 ceph-mon[103001]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:29.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:29 smithi017 ceph-mon[38255]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:31.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:31 smithi017 ceph-mon[103001]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:31.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:31 smithi017 ceph-mon[38255]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:31 smithi189 ceph-mon[33294]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:31.745 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:19:32.479 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/3803582077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3803582077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:32.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:19:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/3803582077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:32 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:19:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[103001]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[103001]: from='client.26639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[103001]: from='client.26645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[103001]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:19:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[38255]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[38255]: from='client.26639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[38255]: from='client.26645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:33 smithi017 ceph-mon[38255]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:19:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:33 smithi189 ceph-mon[33294]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:33 smithi189 ceph-mon[33294]: from='client.26639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:33 smithi189 ceph-mon[33294]: from='client.26645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:33 smithi189 ceph-mon[33294]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:19:34.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:34.728 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:19:34.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: Upgrade: Updating mon.c 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: Deploying daemon mon.c on smithi017 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2786888526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: Upgrade: Updating mon.c 2024-09-13T14:19:34.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: Deploying daemon mon.c on smithi017 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:34.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:34 smithi017 ceph-mon[38255]: from='client.? 172.21.15.17:0/2786888526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: Upgrade: Updating mon.c 2024-09-13T14:19:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: Deploying daemon mon.c on smithi017 2024-09-13T14:19:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:19:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:34 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/2786888526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:19:35.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:35 smithi017 systemd[1]: Stopping Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:19:35.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:35 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c[38232]: 2024-09-13T14:19:35.508+0000 7fd81d251640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:19:35.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:35 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c[38232]: 2024-09-13T14:19:35.508+0000 7fd81d251640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:19:35.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:35 smithi017 podman[107342]: 2024-09-13 14:19:35.539274434 +0000 UTC m=+0.154331286 container died 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:19:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 podman[107342]: 2024-09-13 14:19:36.007117903 +0000 UTC m=+0.622174788 container cleanup 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T14:19:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 bash[107342]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:19:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 podman[107360]: 2024-09-13 14:19:36.359348529 +0000 UTC m=+0.816860632 container remove 67f2b6c9b084a3974e0d32c8dc97831abb6f48d254da7b3eefb78b2feafaad5b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:19:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:19:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 systemd[1]: Stopped Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:19:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.c.service: Consumed 8.346s CPU time. 2024-09-13T14:19:37.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:36 smithi017 systemd[1]: Starting Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:19:37.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 podman[107496]: 2024-09-13 14:19:36.917182368 +0000 UTC m=+0.017037096 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:37.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 podman[107496]: 2024-09-13 14:19:37.027177871 +0000 UTC m=+0.127032596 container create c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:19:37.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:37.696 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:37] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.51.0" 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 podman[107496]: 2024-09-13 14:19:37.30475406 +0000 UTC m=+0.404608785 container init c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 podman[107496]: 2024-09-13 14:19:37.309897192 +0000 UTC m=+0.409751917 container start c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: pidfile_write: ignore empty --pid-file 2024-09-13T14:19:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: load: jerasure load: lrc 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Git sha 0 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: DB SUMMARY 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: DB Session ID: Y4J29TX43Q44NW5SSWJD 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 4061774 ; 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:19:37.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.env: 0x562e81ef8160 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.info_log: 0x562e83d7f880 2024-09-13T14:19:37.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:19:37.699 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.statistics: (nil) 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.use_fsync: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:19:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.db_log_dir: 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.wal_dir: 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.write_buffer_manager: 0x562e83d82960 2024-09-13T14:19:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.unordered_write: 0 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:19:37.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.row_cache: None 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.wal_filter: None 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.wal_compression: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:19:37.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:19:37.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_open_files: -1 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Compression algorithms supported: 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kZSTD supported: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:19:37.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.merge_operator: 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_filter: None 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:19:37.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562e83d7f4e0) 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 4 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x562e83d72850 2024-09-13T14:19:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2024-09-13T14:19:37.709 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: bloomfilter 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 5 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_auto_readahead_size: 262144 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: prepopulate_block_cache: 0 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:19:37.710 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression: NoCompression 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.num_levels: 7 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:19:37.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:19:37.712 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:19:37.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:19:37.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.ttl: 2592000 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:19:37.715 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12921, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:19:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 37d4083c-f79a-49bf-8b63-e1fabc386720 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237177337442, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237177347762, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2590149, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12926, "largest_seqno": 14212, "table_properties": {"data_size": 2584053, "index_size": 3707, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 11940, "raw_average_key_size": 22, "raw_value_size": 2572641, "raw_average_value_size": 4900, "num_data_blocks": 167, "num_entries": 525, "num_filter_entries": 525, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237177, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "37d4083c-f79a-49bf-8b63-e1fabc386720", "db_session_id": "Y4J29TX43Q44NW5SSWJD", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237177347921, "job": 1, "event": "recovery_finished"} 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562e83da2e00 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: DB pointer 0x562e83ea2000 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2024-09-13T14:19:37.717 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: L6 1/0 7.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 2/0 9.93 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:37.718 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 574.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:19:37.719 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 185.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 185.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache BinnedLRUCache@0x562e83d72850#7 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: starting mon.c rank 2 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???) e3 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).mds e9 new map 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).mds e9 print_map 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: e9 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:19:37.720 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: 1 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: Filesystem 'foo' (1) 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: fs_name foo 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: epoch 8 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: created 2024-09-13T14:13:41.125713+0000 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: modified 2024-09-13T14:14:00.359832+0000 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: tableserver 0 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: root 0 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_timeout 60 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_autoclose 300 2024-09-13T14:19:37.721 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_file_size 1099511627776 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_xattr_size 65536 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: required_client_features {} 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure 0 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure_osd_epoch 0 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_mds 1 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: in 0 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: up {0=24539} 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: failed 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: damaged 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: stopped 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_pools [4] 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_pool 3 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: inline_data disabled 2024-09-13T14:19:37.722 INFO:journalctl@ceph.mon.c.smithi017.stdout: balancer 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: bal_rank_mask -1 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: standby_count_wanted 1 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi017.ldrmqj{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: Standby daemons: 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi189.ndtccr{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:37.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 ceph-mon[107556]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-13T14:19:37.724 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 bash[107496]: c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a 2024-09-13T14:19:37.724 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:37 smithi017 systemd[1]: Started Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:19:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:42 smithi017 ceph-mon[103001]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:42 smithi017 ceph-mon[103001]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:42 smithi189 ceph-mon[33294]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:42 smithi189 ceph-mon[33294]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: mon.c calling monitor election 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: mon.a calling monitor election 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: mon.b calling monitor election 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: monmap epoch 3 2024-09-13T14:19:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: min_mon_release 17 (quincy) 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: election_strategy: 1 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:19:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: mon.c calling monitor election 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: mon.a calling monitor election 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: mon.b calling monitor election 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: monmap epoch 3 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: min_mon_release 17 (quincy) 2024-09-13T14:19:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: election_strategy: 1 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:19:43.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:43 smithi017 ceph-mon[107556]: overall HEALTH_OK 2024-09-13T14:19:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: mon.c calling monitor election 2024-09-13T14:19:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: mon.a calling monitor election 2024-09-13T14:19:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: mon.b calling monitor election 2024-09-13T14:19:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: monmap epoch 3 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: last_changed 2024-09-13T14:08:29.611288+0000 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: min_mon_release 17 (quincy) 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: election_strategy: 1 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: Cluster is now healthy 2024-09-13T14:19:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:43 smithi189 ceph-mon[33294]: overall HEALTH_OK 2024-09-13T14:19:44.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:44 smithi017 ceph-mon[103001]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:44 smithi017 ceph-mon[103001]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:44 smithi017 ceph-mon[107556]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:44 smithi017 ceph-mon[107556]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:44 smithi189 ceph-mon[33294]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:44 smithi189 ceph-mon[33294]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:45 smithi017 ceph-mon[103001]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-13T14:19:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:45 smithi017 ceph-mon[107556]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-13T14:19:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:45 smithi189 ceph-mon[33294]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-13T14:19:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:19:47.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:19:47.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:47] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.51.0" 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:19:47.358+0000 7f90e4868640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/59311408' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/59311408' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: from='client.? 172.21.15.17:0/59311408' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:19:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:47 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:48.145 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:48 smithi017 ceph-mon[103001]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:48 smithi017 ceph-mon[107556]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:48 smithi189 ceph-mon[33294]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:50 smithi017 ceph-mon[103001]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:50 smithi017 ceph-mon[107556]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:50 smithi189 ceph-mon[33294]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (3m) 5s ago 7m 18.9M - 0.27.0 11f11916f8cd ceb612349939 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (2m) 25s ago 7m 88.0M - 9.4.7 954c08fa6188 d0c012ef79a6 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (5m) 5s ago 5m 17.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (5m) 25s ago 5m 13.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:19:51.709 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443,9283,8765 running (2m) 25s ago 11m 453M - 19.3.0-4697-gd58f7306 fec07016a2b0 e2be48bfd3a8 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (2m) 5s ago 12m 507M - 19.3.0-4697-gd58f7306 fec07016a2b0 b59080328aea 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (108s) 5s ago 12m 40.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c2de9b169292 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (11m) 25s ago 11m 49.7M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 6439c9112cda 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (14s) 5s ago 11m 20.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c1ddeb604cac 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (2m) 5s ago 6m 11.1M - 1.7.0 72c9c2088986 fce992870ef4 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (2m) 25s ago 6m 9.89M - 1.7.0 72c9c2088986 68ff3e2b1910 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (10m) 5s ago 10m 48.6M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 61ff807d7240 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (10m) 5s ago 10m 53.1M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b8f0ae12aacb 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (10m) 5s ago 10m 46.3M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2e705cedcfaf 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (9m) 5s ago 9m 44.0M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 81c4e9274ff0 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (9m) 25s ago 9m 47.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (8m) 25s ago 8m 46.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (8m) 25s ago 8m 45.9M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (8m) 25s ago 8m 53.0M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:19:51.710 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (2m) 25s ago 7m 46.3M - 2.51.0 1d3b7f56885b 770a1c26888c 2024-09-13T14:19:52.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:52.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:19:52.176 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:52 smithi189 ceph-mon[33294]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:19:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[103001]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:19:52.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:52 smithi017 ceph-mon[107556]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:19:52.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:19:52.811 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:53.181 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:53.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:53.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:53.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:53.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:53.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:19:53.204 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:53 smithi189 ceph-mon[33294]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:19:54.193 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 ceph-mon[33294]: Upgrade: Updating mon.b 2024-09-13T14:19:54.193 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 ceph-mon[33294]: Deploying daemon mon.b on smithi189 2024-09-13T14:19:54.193 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:54.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:54 smithi017 ceph-mon[103001]: Upgrade: Updating mon.b 2024-09-13T14:19:54.240 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:19:54 smithi017 ceph-mon[103001]: Deploying daemon mon.b on smithi189 2024-09-13T14:19:54.240 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:54 smithi017 ceph-mon[107556]: Upgrade: Updating mon.b 2024-09-13T14:19:54.240 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:19:54 smithi017 ceph-mon[107556]: Deploying daemon mon.b on smithi189 2024-09-13T14:19:54.448 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 systemd[1]: Stopping Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:19:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b[33271]: 2024-09-13T14:19:54.447+0000 7f8fc4e93640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:19:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b[33271]: 2024-09-13T14:19:54.447+0000 7f8fc4e93640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:19:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:54 smithi189 podman[90802]: 2024-09-13 14:19:54.683204148 +0000 UTC m=+0.346728043 container died 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True) 2024-09-13T14:19:55.233 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 podman[90802]: 2024-09-13 14:19:55.056516823 +0000 UTC m=+0.720040718 container cleanup 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T14:19:55.233 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 bash[90802]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:19:55.491 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 podman[90819]: 2024-09-13 14:19:55.232969106 +0000 UTC m=+0.548760977 container remove 6439c9112cda6c946ed988ff43312f3304622d6a58365844c108244d9bcb42bb (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T14:19:55.491 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T14:19:55.491 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 systemd[1]: Stopped Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:19:55.491 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.b.service: Consumed 8.502s CPU time. 2024-09-13T14:19:55.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 systemd[1]: Starting Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:19:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 podman[90954]: 2024-09-13 14:19:55.788627173 +0000 UTC m=+0.016174992 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:19:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:55 smithi189 podman[90954]: 2024-09-13 14:19:55.909400532 +0000 UTC m=+0.136948349 container create 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:19:56.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:56.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:56.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 podman[90954]: 2024-09-13 14:19:56.120163055 +0000 UTC m=+0.347710869 container init 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 podman[90954]: 2024-09-13 14:19:56.12468385 +0000 UTC m=+0.352231665 container start 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: pidfile_write: ignore empty --pid-file 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: load: jerasure load: lrc 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Git sha 0 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: DB SUMMARY 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: DB Session ID: WPF0RITKBGEM74YYF6MU 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:19:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 4865501 ; 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.env: 0x55751d78e160 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.info_log: 0x55751e237880 2024-09-13T14:19:56.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:19:56.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.statistics: (nil) 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.use_fsync: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:19:56.509 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.db_log_dir: 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.wal_dir: 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.write_buffer_manager: 0x55751e23a960 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:19:56.510 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.unordered_write: 0 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.row_cache: None 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.wal_filter: None 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:19:56.511 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.wal_compression: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:19:56.512 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:19:56.513 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_open_files: -1 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Compression algorithms supported: 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kZSTD supported: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:19:56.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.merge_operator: 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_filter: None 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55751e2374e0) 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:19:56.515 INFO:journalctl@ceph.mon.b.smithi189.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_type: 0 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: data_block_index_type: 0 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_shortening: 1 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: checksum: 4 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: no_block_cache: 0 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache: 0x55751e22a850 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_options: 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: capacity : 536870912 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: num_shard_bits : 4 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: strict_capacity_limit : 0 2024-09-13T14:19:56.516 INFO:journalctl@ceph.mon.b.smithi189.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_cache_compressed: (nil) 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: persistent_cache: (nil) 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_size: 4096 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_size_deviation: 10 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_restart_interval: 16 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: index_block_restart_interval: 1 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: metadata_block_size: 4096 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: partition_filters: 0 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: use_delta_encoding: 1 2024-09-13T14:19:56.517 INFO:journalctl@ceph.mon.b.smithi189.stdout: filter_policy: bloomfilter 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: whole_key_filtering: 1 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: verify_compression: 0 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: format_version: 5 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: enable_index_compression: 1 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: block_align: 0 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: max_auto_readahead_size: 262144 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: prepopulate_block_cache: 0 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:19:56.518 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression: NoCompression 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.num_levels: 7 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:19:56.519 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:19:56.520 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:19:56.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:19:56.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:19:56.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.ttl: 2592000 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:19:56.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12993, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fe899aa2-5a2d-4184-b423-6aa05d9b8818 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237196151598, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237196162587, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3063793, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12998, "largest_seqno": 14597, "table_properties": {"data_size": 3056667, "index_size": 4481, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14578, "raw_average_key_size": 22, "raw_value_size": 3042852, "raw_average_value_size": 4754, "num_data_blocks": 203, "num_entries": 640, "num_filter_entries": 640, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237196, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fe899aa2-5a2d-4184-b423-6aa05d9b8818", "db_session_id": "WPF0RITKBGEM74YYF6MU", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237196162730, "job": 1, "event": "recovery_finished"} 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:19:56.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55751e25ae00 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: DB pointer 0x55751e35a000 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** DB Stats ** 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:19:56.526 INFO:journalctl@ceph.mon.b.smithi189.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: L0 1/0 2.92 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 692.4 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: L6 1/0 7.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: Sum 2/0 10.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 692.4 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 692.4 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** Compaction Stats [default] ** 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 692.4 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:19:56.527 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Cumulative compaction: 0.00 GB write, 183.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Interval compaction: 0.00 GB write, 183.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Block cache BinnedLRUCache@0x55751e22a850#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:19:56.528 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: starting mon.b rank 1 at public addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] at bind addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???) e3 preinit fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).mds e9 new map 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).mds e9 print_map 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: e9 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: legacy client fscid: 1 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: Filesystem 'foo' (1) 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: fs_name foo 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: epoch 8 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: created 2024-09-13T14:13:41.125713+0000 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: modified 2024-09-13T14:14:00.359832+0000 2024-09-13T14:19:56.529 INFO:journalctl@ceph.mon.b.smithi189.stdout: tableserver 0 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: root 0 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: session_timeout 60 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: session_autoclose 300 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: max_file_size 1099511627776 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: max_xattr_size 65536 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: required_client_features {} 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: last_failure 0 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: last_failure_osd_epoch 0 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: max_mds 1 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: in 0 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: up {0=24539} 2024-09-13T14:19:56.530 INFO:journalctl@ceph.mon.b.smithi189.stdout: failed 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: damaged 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: stopped 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: data_pools [4] 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: metadata_pool 3 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: inline_data disabled 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: balancer 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: bal_rank_mask -1 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: standby_count_wanted 1 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: [mds.foo.smithi017.ldrmqj{0:24539} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/1932644911,v1:172.21.15.17:6835/1932644911] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: Standby daemons: 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: 2024-09-13T14:19:56.531 INFO:journalctl@ceph.mon.b.smithi189.stdout: [mds.foo.smithi189.ndtccr{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.189:6832/3236772480,v1:172.21.15.189:6833/3236772480] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-mon[90992]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 bash[90954]: 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 2024-09-13T14:19:56.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:19:56 smithi189 systemd[1]: Started Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:19:57.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:19:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:19:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:19:57.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:19:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:19:57] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.51.0" 2024-09-13T14:20:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:01.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:01 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:20:01.176+0000 7f90e4868640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:20:02.727 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:20:03.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:20:03.736 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mon.c calling monitor election 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mon.b calling monitor election 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: min_mon_release 19 (squid) 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: election_strategy: 1 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:03.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: overall HEALTH_OK 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mon.a calling monitor election 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: monmap epoch 4 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: min_mon_release 19 (squid) 2024-09-13T14:20:03.987 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: election_strategy: 1 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: overall HEALTH_OK 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mon.c calling monitor election 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mon.b calling monitor election 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: min_mon_release 19 (squid) 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: election_strategy: 1 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:03.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mon.a calling monitor election 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: monmap epoch 4 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: min_mon_release 19 (squid) 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: election_strategy: 1 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:03.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:03.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mon.c calling monitor election 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mon.b calling monitor election 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: min_mon_release 19 (squid) 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: election_strategy: 1 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: overall HEALTH_OK 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mon.a calling monitor election 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: monmap epoch 4 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: last_changed 2024-09-13T14:19:57.689278+0000 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: created 2024-09-13T14:06:53.813281+0000 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: min_mon_release 19 (squid) 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: election_strategy: 1 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 1: [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon.b 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: fsmap foo:1 {0=foo.smithi017.ldrmqj=up:active} 1 up:standby 2024-09-13T14:20:04.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: mgrmap e40: y(active, since 111s), standbys: x 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: overall HEALTH_OK 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:04.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:04.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:04.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:04 smithi017 ceph-mon[103001]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:04.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:04 smithi017 ceph-mon[107556]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:04 smithi189 ceph-mon[90992]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:07.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:07 smithi189 ceph-mon[90992]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:07.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:07.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:07.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[103001]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:07.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:07] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.51.0" 2024-09-13T14:20:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[107556]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:08 smithi189 ceph-mon[90992]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[103001]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[107556]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:10.404 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:10 smithi189 ceph-mon[90992]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:10 smithi017 ceph-mon[103001]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:10 smithi017 ceph-mon[107556]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:12.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:11 smithi189 ceph-mon[90992]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:20:12.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[107556]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:12.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:11 smithi017 ceph-mon[103001]: Reconfiguring daemon mon.a on smithi017 2024-09-13T14:20:12.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:12.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:12.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[103001]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:20:12.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:12.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:12.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:12.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:12 smithi017 ceph-mon[107556]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:20:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:12 smithi189 ceph-mon[90992]: Reconfiguring daemon mgr.y on smithi017 2024-09-13T14:20:14.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[103001]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:14 smithi017 ceph-mon[107556]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:20:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:20:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:15.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:14 smithi189 ceph-mon[90992]: Reconfiguring daemon mon.c on smithi017 2024-09-13T14:20:16.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:16.006 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.0 on smithi017 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:20:16.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:20:16.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:16.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:16 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.0 on smithi017 2024-09-13T14:20:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:20:16.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:20:16.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:16.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.0 on smithi017 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[107556]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[103001]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.180 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:20:17.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:17.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:16 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:17 smithi189 ceph-mon[90992]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:20:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:17] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.51.0" 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.1 on smithi017 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.1 on smithi017 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:20:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.1 on smithi017 2024-09-13T14:20:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:20:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[103001]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[103001]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.2 on smithi017 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[107556]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[107556]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:20:19.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:19 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.2 on smithi017 2024-09-13T14:20:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:19 smithi189 ceph-mon[90992]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:19 smithi189 ceph-mon[90992]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:20:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:19 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.2 on smithi017 2024-09-13T14:20:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:20.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:20.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:20.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:20:20.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:20:20.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.3 on smithi017 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[103001]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.3 on smithi017 2024-09-13T14:20:20.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:20 smithi017 ceph-mon[107556]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:20:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:20:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.3 on smithi017 2024-09-13T14:20:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:20 smithi189 ceph-mon[90992]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:22.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: Reconfiguring mds.foo.smithi017.ldrmqj (monmap changed)... 2024-09-13T14:20:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:21 smithi189 ceph-mon[90992]: Reconfiguring daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: Reconfiguring mds.foo.smithi017.ldrmqj (monmap changed)... 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:22.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[103001]: Reconfiguring daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: Reconfiguring mds.foo.smithi017.ldrmqj (monmap changed)... 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ldrmqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:22.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:21 smithi017 ceph-mon[107556]: Reconfiguring daemon mds.foo.smithi017.ldrmqj on smithi017 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:23.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:23.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:23.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:20:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:24.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:20:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:20:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: Reconfiguring daemon mon.b on smithi189 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:20:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:24.454 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:24 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:25.108 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:24 smithi189 ceph-mon[90992]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:25.108 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:24 smithi189 ceph-mon[90992]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:20:25.108 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:24 smithi189 ceph-mon[90992]: Reconfiguring daemon mgr.x on smithi189 2024-09-13T14:20:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[103001]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[103001]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:20:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[103001]: Reconfiguring daemon mgr.x on smithi189 2024-09-13T14:20:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[107556]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[107556]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:20:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:24 smithi017 ceph-mon[107556]: Reconfiguring daemon mgr.x on smithi189 2024-09-13T14:20:25.828 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:25.828 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.4 on smithi189 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:26 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.4 on smithi189 2024-09-13T14:20:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:20:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:20:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.4 on smithi189 2024-09-13T14:20:26.952 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:26 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:27.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:27.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:20:27.240 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:27.240 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:27.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:27] "GET /metrics HTTP/1.1" 200 33989 "" "Prometheus/2.51.0" 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.5 on smithi189 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:20:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.5 on smithi189 2024-09-13T14:20:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:20:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.5 on smithi189 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:20:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[107556]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[107556]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.6 on smithi189 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[103001]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[103001]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:20:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:29 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.6 on smithi189 2024-09-13T14:20:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:29 smithi189 ceph-mon[90992]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:29 smithi189 ceph-mon[90992]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:20:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:29 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.6 on smithi189 2024-09-13T14:20:30.240 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: Reconfiguring daemon osd.7 on smithi189 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[107556]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: Reconfiguring daemon osd.7 on smithi189 2024-09-13T14:20:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:30 smithi017 ceph-mon[103001]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:30.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:30.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:20:30.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:20:30.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:30.953 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: Reconfiguring daemon osd.7 on smithi189 2024-09-13T14:20:30.953 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:30 smithi189 ceph-mon[90992]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:32.442 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: Reconfiguring mds.foo.smithi189.ndtccr (monmap changed)... 2024-09-13T14:20:32.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:32.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:32.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:32 smithi189 ceph-mon[90992]: Reconfiguring daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: Reconfiguring mds.foo.smithi189.ndtccr (monmap changed)... 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[103001]: Reconfiguring daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:20:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: Reconfiguring mds.foo.smithi189.ndtccr (monmap changed)... 2024-09-13T14:20:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.ndtccr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:20:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:32 smithi017 ceph-mon[107556]: Reconfiguring daemon mds.foo.smithi189.ndtccr on smithi189 2024-09-13T14:20:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:20:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:20:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:20:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:20:34.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:35 smithi017 ceph-mon[103001]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:35 smithi017 ceph-mon[107556]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:35 smithi189 ceph-mon[90992]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:36.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:37 smithi189 ceph-mon[90992]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:37.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:37.348 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:37 smithi017 ceph-mon[103001]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:37.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:37 smithi017 ceph-mon[107556]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:37] "GET /metrics HTTP/1.1" 200 33989 "" "Prometheus/2.51.0" 2024-09-13T14:20:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:39 smithi017 ceph-mon[103001]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:39 smithi017 ceph-mon[107556]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:39 smithi189 ceph-mon[90992]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:41 smithi017 ceph-mon[103001]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:41 smithi017 ceph-mon[107556]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:41 smithi189 ceph-mon[90992]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:43 smithi017 ceph-mon[103001]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:43 smithi017 ceph-mon[107556]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:43 smithi189 ceph-mon[90992]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:44.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:45 smithi017 ceph-mon[103001]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:45 smithi017 ceph-mon[107556]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:45 smithi189 ceph-mon[90992]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:47 smithi189 ceph-mon[90992]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:47.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:47.348 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:47 smithi017 ceph-mon[103001]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:47.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:47 smithi017 ceph-mon[107556]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:47] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.51.0" 2024-09-13T14:20:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:49 smithi017 ceph-mon[103001]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:49 smithi017 ceph-mon[107556]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:49 smithi189 ceph-mon[90992]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:51 smithi017 ceph-mon[103001]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:51 smithi017 ceph-mon[107556]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:51 smithi189 ceph-mon[90992]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:53 smithi017 ceph-mon[103001]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:53 smithi017 ceph-mon[107556]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:53 smithi189 ceph-mon[90992]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:54.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:54.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:55 smithi017 ceph-mon[103001]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:55 smithi017 ceph-mon[107556]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:55 smithi189 ceph-mon[90992]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:57.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:57 smithi189 ceph-mon[90992]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:57.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:57.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:20:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:20:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:20:57.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:57 smithi017 ceph-mon[103001]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:57.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:57.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:57 smithi017 ceph-mon[107556]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:57.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:20:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:20:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:20:57] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.51.0" 2024-09-13T14:20:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:20:59 smithi017 ceph-mon[103001]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:20:59 smithi017 ceph-mon[107556]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:20:59 smithi189 ceph-mon[90992]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:01 smithi017 ceph-mon[103001]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:01 smithi017 ceph-mon[107556]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:01.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:01 smithi189 ceph-mon[90992]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:03 smithi017 ceph-mon[103001]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:03 smithi017 ceph-mon[107556]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:03 smithi189 ceph-mon[90992]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:04.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:05.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:05 smithi017 ceph-mon[103001]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:05.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:05 smithi017 ceph-mon[107556]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:05 smithi189 ceph-mon[90992]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:06.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:06.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:21:06.336 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:07.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:07 smithi189 ceph-mon[90992]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:07.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:07.348 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:07 smithi017 ceph-mon[103001]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:07.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:07 smithi017 ceph-mon[107556]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:07] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.51.0" 2024-09-13T14:21:08.442 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[103001]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[107556]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:09.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:09.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for all mons to upgrade' 2024-09-13T14:21:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:09 smithi189 ceph-mon[90992]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:09.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:09.852 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:10.266 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:10 smithi017 ceph-mon[103001]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:10.266 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:10 smithi017 ceph-mon[103001]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:10.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:10 smithi017 ceph-mon[107556]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:10.267 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:10 smithi017 ceph-mon[107556]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:10 smithi189 ceph-mon[90992]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:10 smithi189 ceph-mon[90992]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:11.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:11 smithi017 ceph-mon[103001]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:11.263 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:11 smithi017 ceph-mon[107556]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:11.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:11 smithi189 ceph-mon[90992]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:11.481 INFO:teuthology.orchestra.run.smithi017.stdout:wait for all mons to upgrade 2024-09-13T14:21:12.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:12.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:12.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:21:12.425 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:12.979 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[103001]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[107556]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:13 smithi189 ceph-mon[90992]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:14.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:14 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:15 smithi017 ceph-mon[103001]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:15 smithi017 ceph-mon[107556]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:15 smithi189 ceph-mon[90992]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.528 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:21:16.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:16.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:15 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:16.168 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/416695696' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:21:16.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/416695696' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:21:16.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:21:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/416695696' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:21:16.692 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:17 smithi189 ceph-mon[90992]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:17.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:16 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:17] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.51.0" 2024-09-13T14:21:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[103001]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[107556]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: Upgrade: Setting container_image for all mon 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: Upgrade: Setting container_image for all crash 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:21:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[103001]: Upgrade: osd.0 is safe to restart 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: Upgrade: Setting container_image for all mon 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:21:18.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: Upgrade: Setting container_image for all crash 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:21:18.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:18 smithi017 ceph-mon[107556]: Upgrade: osd.0 is safe to restart 2024-09-13T14:21:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: Upgrade: Setting container_image for all mon 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: Upgrade: Setting container_image for all crash 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:21:18.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:21:18.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:18 smithi189 ceph-mon[90992]: Upgrade: osd.0 is safe to restart 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (4m) 2s ago 8m 18.4M - 0.27.0 11f11916f8cd ceb612349939 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (4m) 6s ago 8m 87.5M - 9.4.7 954c08fa6188 d0c012ef79a6 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (7m) 2s ago 7m 18.1M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (7m) 6s ago 7m 14.5M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:21:18.888 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443,9283,8765 running (3m) 6s ago 12m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 e2be48bfd3a8 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (3m) 2s ago 14m 510M - 19.3.0-4697-gd58f7306 fec07016a2b0 b59080328aea 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (3m) 2s ago 14m 44.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c2de9b169292 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (83s) 6s ago 12m 35.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 37d58338d569 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (101s) 2s ago 12m 36.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c1ddeb604cac 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (4m) 2s ago 8m 11.3M - 1.7.0 72c9c2088986 fce992870ef4 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (4m) 6s ago 8m 12.1M - 1.7.0 72c9c2088986 68ff3e2b1910 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (12m) 2s ago 12m 49.1M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 61ff807d7240 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (11m) 2s ago 11m 53.6M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a b8f0ae12aacb 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (11m) 2s ago 11m 46.9M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2e705cedcfaf 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (11m) 2s ago 11m 44.7M 2481M 17.2.7-1627-gbbdadfa3 a994b00deb3a 81c4e9274ff0 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (10m) 6s ago 10m 48.6M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (10m) 6s ago 10m 47.4M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (9m) 6s ago 9m 46.6M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:21:18.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (9m) 6s ago 9m 53.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:21:18.890 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (3m) 6s ago 9m 51.9M - 2.51.0 1d3b7f56885b 770a1c26888c 2024-09-13T14:21:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:19 smithi017 ceph-mon[107556]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:19 smithi017 ceph-mon[103001]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:19 smithi189 ceph-mon[90992]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:20.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:21:20.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[103001]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:20.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:20.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:21:20.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:20.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[107556]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:20.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:20.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:21:20.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:20 smithi189 ceph-mon[90992]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:21:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:20.918 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[103001]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[103001]: Upgrade: Updating osd.0 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[103001]: Deploying daemon osd.0 on smithi017 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[107556]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[107556]: Upgrade: Updating osd.0 2024-09-13T14:21:21.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:21 smithi017 ceph-mon[107556]: Deploying daemon osd.0 on smithi017 2024-09-13T14:21:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:21 smithi189 ceph-mon[90992]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:21 smithi189 ceph-mon[90992]: Upgrade: Updating osd.0 2024-09-13T14:21:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:21 smithi189 ceph-mon[90992]: Deploying daemon osd.0 on smithi017 2024-09-13T14:21:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:23 smithi017 ceph-mon[103001]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:23 smithi017 ceph-mon[107556]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:23 smithi189 ceph-mon[90992]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:24.045 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:23 smithi017 systemd[1]: Stopping Ceph osd.0 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:24.360 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:24.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:24 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:24.647 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:24 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[42619]: 2024-09-13T14:21:24.307+0000 7f61b63d5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:21:24.647 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:24 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[42619]: 2024-09-13T14:21:24.307+0000 7f61b63d5640 -1 osd.0 82 *** Got signal Terminated *** 2024-09-13T14:21:24.647 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:24 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[42619]: 2024-09-13T14:21:24.307+0000 7f61b63d5640 -1 osd.0 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:21:25.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:21:25.343 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[103001]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[103001]: osd.0 marked itself down and dead 2024-09-13T14:21:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[103001]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[103001]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[107556]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[107556]: osd.0 marked itself down and dead 2024-09-13T14:21:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[107556]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[107556]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:25.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-mon[90992]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-mon[90992]: osd.0 marked itself down and dead 2024-09-13T14:21:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-mon[90992]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-mon[90992]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:25.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:26.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:25 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:26.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:26 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:21:26.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:26 smithi017 ceph-mon[103001]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:21:26.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:26 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:21:26.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:26 smithi017 ceph-mon[107556]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:21:26.435 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:26 smithi017 podman[115339]: 2024-09-13 14:21:26.12047156 +0000 UTC m=+1.921450964 container died 61ff807d7240556fb2645aae79b477a7c080d03e2ceb3f562b0f4aead593d7f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:21:26.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:26 smithi017 podman[115339]: 2024-09-13 14:21:26.522872576 +0000 UTC m=+2.323851934 container cleanup 61ff807d7240556fb2645aae79b477a7c080d03e2ceb3f562b0f4aead593d7f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T14:21:26.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:26 smithi017 bash[115339]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:21:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:26 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:21:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:26 smithi189 ceph-mon[90992]: osdmap e83: 8 total, 7 up, 8 in 2024-09-13T14:21:27.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:27 smithi017 podman[115502]: 2024-09-13 14:21:27.110275535 +0000 UTC m=+1.307644467 container remove 61ff807d7240556fb2645aae79b477a7c080d03e2ceb3f562b0f4aead593d7f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T14:21:27.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:26 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:27.489 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:27] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.51.0" 2024-09-13T14:21:27.489 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[107556]: pgmap v101: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[107556]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[103001]: pgmap v101: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[103001]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:27.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:27 smithi189 ceph-mon[90992]: pgmap v101: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:27 smithi189 ceph-mon[90992]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T14:21:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:27.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:28.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:27 smithi017 podman[115655]: 2024-09-13 14:21:27.838806723 +0000 UTC m=+0.016676886 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:28.201 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:27 smithi017 podman[115655]: 2024-09-13 14:21:27.948149446 +0000 UTC m=+0.126019652 container create 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T14:21:28.201 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 podman[115655]: 2024-09-13 14:21:28.16726343 +0000 UTC m=+0.345133609 container init 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T14:21:28.201 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 podman[115655]: 2024-09-13 14:21:28.171996743 +0000 UTC m=+0.349866907 container start 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T14:21:28.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:28 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:21:28.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:28 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:21:28.672 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 podman[115655]: 2024-09-13 14:21:28.283783336 +0000 UTC m=+0.461653514 container attach 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T14:21:28.672 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 podman[115655]: 2024-09-13 14:21:28.370308121 +0000 UTC m=+0.548178285 container died 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T14:21:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:28 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:21:29.349 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 podman[115699]: 2024-09-13 14:21:28.954095961 +0000 UTC m=+0.577256436 container remove 8767039176f6308a064009eb7cd21c3d6593c83f18e8f35c2a925c8e6edc2d03 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:21:29.350 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:21:29.350 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 systemd[1]: Stopped Ceph osd.0 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:29.350 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:28 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.0.service: Consumed 5.890s CPU time. 2024-09-13T14:21:29.350 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 systemd[1]: Starting Ceph osd.0 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:21:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-mon[103001]: pgmap v103: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-mon[107556]: pgmap v103: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:29.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 podman[115803]: 2024-09-13 14:21:29.317203533 +0000 UTC m=+0.016860720 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:29.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 podman[115803]: 2024-09-13 14:21:29.438749148 +0000 UTC m=+0.138406386 container create c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:21:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:29 smithi189 ceph-mon[90992]: pgmap v103: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:29.952 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 podman[115803]: 2024-09-13 14:21:29.699696791 +0000 UTC m=+0.399353966 container init c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True) 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 podman[115803]: 2024-09-13 14:21:29.705327785 +0000 UTC m=+0.404984972 container start c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 podman[115803]: 2024-09-13 14:21:29.799523856 +0000 UTC m=+0.499181042 container attach c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.953 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.954 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:21:29.954 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:21:29.954 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:21:30.237 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:21:30.237 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:29 smithi017 bash[115803]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:21:30.237 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:21:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 bash[115803]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 bash[115803]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 bash[115803]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 bash[115803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate[115818]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 bash[115803]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:21:30.563 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 podman[115803]: 2024-09-13 14:21:30.267716647 +0000 UTC m=+0.967373832 container died c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T14:21:31.192 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:30 smithi017 podman[115959]: 2024-09-13 14:21:30.836745736 +0000 UTC m=+0.556224892 container remove c346b0629c018eb81858a8b064cf9df9d87051bf84c877155dca7626677951d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:21:31.192 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 podman[116024]: 2024-09-13 14:21:30.96059127 +0000 UTC m=+0.017638640 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:21:31.192 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 podman[116024]: 2024-09-13 14:21:31.071186978 +0000 UTC m=+0.128234392 container create b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:21:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:31 smithi017 ceph-mon[103001]: pgmap v104: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:31 smithi017 ceph-mon[107556]: pgmap v104: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:31.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 podman[116024]: 2024-09-13 14:21:31.281952011 +0000 UTC m=+0.338999380 container init b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T14:21:31.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 podman[116024]: 2024-09-13 14:21:31.287524699 +0000 UTC m=+0.344572069 container start b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:21:31.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 ceph-osd[116061]: -- 172.21.15.17:0/2940745647 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d3adfbda40 con 0x55d3aed8a400 2024-09-13T14:21:31.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 bash[116024]: b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe 2024-09-13T14:21:31.702 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:31 smithi017 systemd[1]: Started Ceph osd.0 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:31.739 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:31 smithi189 ceph-mon[90992]: pgmap v104: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:32 smithi189 ceph-mon[90992]: pgmap v105: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[107556]: pgmap v105: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:32.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-mon[103001]: pgmap v105: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:33.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:32 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T14:21:32.925+0000 7f02e9c4a640 -1 Falling back to public interface 2024-09-13T14:21:34.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:34.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:21:34.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:34 smithi189 ceph-mon[90992]: pgmap v106: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:35.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:34 smithi017 ceph-mon[103001]: pgmap v106: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:35.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:34 smithi017 ceph-mon[107556]: pgmap v106: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:36.001 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:36.001 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:36.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:36 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T14:21:36.468+0000 7f02e9c4a640 -1 osd.0 82 log_to_monitors true 2024-09-13T14:21:37.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-mon[103001]: pgmap v107: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:37.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-mon[103001]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:21:37.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-mon[107556]: pgmap v107: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:37.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-mon[107556]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:21:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:37 smithi189 ceph-mon[90992]: pgmap v107: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:37 smithi189 ceph-mon[90992]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:21:37.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:37.549 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:37] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.51.0" 2024-09-13T14:21:37.549 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 14:21:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T14:21:37.288+0000 7f02dd9ec640 -1 osd.0 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[103001]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[103001]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[103001]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[103001]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[107556]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[107556]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:21:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[107556]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:21:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:38 smithi017 ceph-mon[107556]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' 2024-09-13T14:21:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:38 smithi189 ceph-mon[90992]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:21:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:38 smithi189 ceph-mon[90992]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T14:21:38.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:38 smithi189 ceph-mon[90992]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:21:38.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:38 smithi189 ceph-mon[90992]: from='osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078]' entity='osd.0' 2024-09-13T14:21:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: pgmap v109: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:21:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078] boot 2024-09-13T14:21:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: pgmap v109: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078] boot 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:39.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: pgmap v109: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: osd.0 [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078] boot 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:21:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:40 smithi189 ceph-mon[90992]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:21:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:40 smithi017 ceph-mon[103001]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:21:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:40 smithi017 ceph-mon[107556]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:21:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:41 smithi189 ceph-mon[90992]: pgmap v112: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:41 smithi017 ceph-mon[103001]: pgmap v112: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:41 smithi017 ceph-mon[107556]: pgmap v112: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[103001]: pgmap v113: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[107556]: pgmap v113: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:43 smithi189 ceph-mon[90992]: pgmap v113: 73 pgs: 18 active+undersized, 8 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-09-13T14:21:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:44 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded) 2024-09-13T14:21:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:44 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:21:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:44 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded) 2024-09-13T14:21:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:44 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:21:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:44 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 8 pgs degraded) 2024-09-13T14:21:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:44 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:21:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:44.146Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T14:21:44.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:45 smithi017 ceph-mon[103001]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:45 smithi017 ceph-mon[107556]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:45 smithi189 ceph-mon[90992]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:46.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:47 smithi189 ceph-mon[90992]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi189\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi189\", job=\"node\", machine=\"x86_64\", nodename=\"smithi189\", release=\"5.14.0-505.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:47 smithi017 ceph-mon[103001]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:47 smithi017 ceph-mon[107556]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:47] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.51.0" 2024-09-13T14:21:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:49 smithi017 ceph-mon[103001]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:49 smithi017 ceph-mon[107556]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:49 smithi189 ceph-mon[90992]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:51 smithi017 ceph-mon[103001]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:51 smithi017 ceph-mon[107556]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:51 smithi189 ceph-mon[90992]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:53 smithi017 ceph-mon[103001]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:53 smithi017 ceph-mon[107556]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:53 smithi189 ceph-mon[90992]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:54.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:55 smithi017 ceph-mon[103001]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:55 smithi017 ceph-mon[107556]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:55 smithi189 ceph-mon[90992]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:56.005 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:21:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:21:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:21:57.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[103001]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:57.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:57.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:57.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[107556]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:57.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:57.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:57.463 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:21:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:21:57] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-09-13T14:21:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:57 smithi189 ceph-mon[90992]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:21:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:21:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:21:59 smithi017 ceph-mon[103001]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:21:59 smithi017 ceph-mon[107556]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:21:59 smithi189 ceph-mon[90992]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:01 smithi017 ceph-mon[103001]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:01 smithi017 ceph-mon[107556]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:01 smithi189 ceph-mon[90992]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:03 smithi017 ceph-mon[103001]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:03 smithi017 ceph-mon[107556]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:03 smithi189 ceph-mon[90992]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:05 smithi017 ceph-mon[103001]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:05 smithi017 ceph-mon[107556]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:05 smithi189 ceph-mon[90992]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:07 smithi017 ceph-mon[103001]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:07 smithi017 ceph-mon[107556]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:07.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:07] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-09-13T14:22:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:07 smithi189 ceph-mon[90992]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:09 smithi017 ceph-mon[103001]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:09 smithi017 ceph-mon[107556]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:09 smithi189 ceph-mon[90992]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:11 smithi017 ceph-mon[103001]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:11 smithi017 ceph-mon[107556]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:11 smithi189 ceph-mon[90992]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:13 smithi017 ceph-mon[103001]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:13 smithi017 ceph-mon[107556]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:13 smithi189 ceph-mon[90992]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:15 smithi017 ceph-mon[103001]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:15 smithi017 ceph-mon[107556]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:15 smithi189 ceph-mon[90992]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:17 smithi017 ceph-mon[103001]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:17 smithi017 ceph-mon[107556]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:17.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:17] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T14:22:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:17 smithi189 ceph-mon[90992]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:19 smithi017 ceph-mon[103001]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:19 smithi017 ceph-mon[107556]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:19 smithi189 ceph-mon[90992]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:21 smithi017 ceph-mon[103001]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:21 smithi017 ceph-mon[107556]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:21 smithi189 ceph-mon[90992]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:23 smithi017 ceph-mon[103001]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:23 smithi017 ceph-mon[107556]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:23 smithi189 ceph-mon[90992]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:25 smithi017 ceph-mon[103001]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:25 smithi017 ceph-mon[107556]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:25 smithi189 ceph-mon[90992]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:27 smithi017 ceph-mon[103001]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:27.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:27] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T14:22:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:27 smithi017 ceph-mon[107556]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:27 smithi189 ceph-mon[90992]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:28.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:22:28.510 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:29 smithi017 ceph-mon[103001]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:29 smithi017 ceph-mon[107556]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:29 smithi189 ceph-mon[90992]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:30.600 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:31.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[107556]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:31.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:31.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[103001]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:31.205 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:31 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:31.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-13T14:22:31.490 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:31 smithi189 ceph-mon[90992]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:31 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:31 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:32 smithi189 ceph-mon[90992]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:32 smithi189 ceph-mon[90992]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:32 smithi017 ceph-mon[103001]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:32 smithi017 ceph-mon[103001]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:32 smithi017 ceph-mon[107556]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:32 smithi017 ceph-mon[107556]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:33.148 INFO:teuthology.orchestra.run.smithi017.stdout:wait for half of osds to upgrade 2024-09-13T14:22:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:33 smithi017 ceph-mon[103001]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:33 smithi017 ceph-mon[107556]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:33 smithi189 ceph-mon[90992]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:33.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:22:34.030 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:36.227 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: Upgrade: osd.1 is safe to restart 2024-09-13T14:22:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/778345018' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: Upgrade: osd.1 is safe to restart 2024-09-13T14:22:36.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/778345018' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: Upgrade: osd.1 is safe to restart 2024-09-13T14:22:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/778345018' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:37] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T14:22:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[103001]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[103001]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[107556]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[107556]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:22:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:37 smithi189 ceph-mon[90992]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:37 smithi189 ceph-mon[90992]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:22:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[107556]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[107556]: Upgrade: Updating osd.1 2024-09-13T14:22:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[107556]: Deploying daemon osd.1 on smithi017 2024-09-13T14:22:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[103001]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[103001]: Upgrade: Updating osd.1 2024-09-13T14:22:38.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:38 smithi017 ceph-mon[103001]: Deploying daemon osd.1 on smithi017 2024-09-13T14:22:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:38 smithi189 ceph-mon[90992]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:38 smithi189 ceph-mon[90992]: Upgrade: Updating osd.1 2024-09-13T14:22:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:38 smithi189 ceph-mon[90992]: Deploying daemon osd.1 on smithi017 2024-09-13T14:22:39.114 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:39.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:39 smithi017 ceph-mon[103001]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:39.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/404583615' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:39.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:39 smithi017 ceph-mon[107556]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:39.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/404583615' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:39 smithi189 ceph-mon[90992]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/404583615' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:40.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:40 smithi017 ceph-mon[103001]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:40.286 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:40 smithi017 ceph-mon[107556]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:40.287 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:40 smithi017 systemd[1]: Stopping Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:40 smithi189 ceph-mon[90992]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:40.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:40 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[48981]: 2024-09-13T14:22:40.284+0000 7f44c9a48640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:22:40.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:40 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[48981]: 2024-09-13T14:22:40.284+0000 7f44c9a48640 -1 osd.1 87 *** Got signal Terminated *** 2024-09-13T14:22:40.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:40 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[48981]: 2024-09-13T14:22:40.284+0000 7f44c9a48640 -1 osd.1 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[103001]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[103001]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[103001]: osd.1 marked itself down and dead 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[107556]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[107556]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:41 smithi017 ceph-mon[107556]: osd.1 marked itself down and dead 2024-09-13T14:22:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:41 smithi189 ceph-mon[90992]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:41 smithi189 ceph-mon[90992]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:41 smithi189 ceph-mon[90992]: osd.1 marked itself down and dead 2024-09-13T14:22:41.917 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:41 smithi017 podman[121203]: 2024-09-13 14:22:41.656032018 +0000 UTC m=+1.495518929 container died b8f0ae12aacb0dd1fcc23e5167c51ee81e3f9071a5aef2749921f1f125dcbe65 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T14:22:42.021 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:42.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121203]: 2024-09-13 14:22:42.00711266 +0000 UTC m=+1.846599559 container cleanup b8f0ae12aacb0dd1fcc23e5167c51ee81e3f9071a5aef2749921f1f125dcbe65 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T14:22:42.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 bash[121203]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:22:42.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121246]: 2024-09-13 14:22:42.30886701 +0000 UTC m=+0.648074105 container remove b8f0ae12aacb0dd1fcc23e5167c51ee81e3f9071a5aef2749921f1f125dcbe65 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa) 2024-09-13T14:22:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:22:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[103001]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T14:22:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3694287933' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[107556]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3694287933' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:42 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:22:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:42 smithi189 ceph-mon[90992]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T14:22:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3694287933' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:42.880 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121389]: 2024-09-13 14:22:42.626842428 +0000 UTC m=+0.104203929 container create 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T14:22:42.880 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121389]: 2024-09-13 14:22:42.538981477 +0000 UTC m=+0.016342988 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:42.880 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121389]: 2024-09-13 14:22:42.879805893 +0000 UTC m=+0.357167407 container init 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T14:22:43.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121389]: 2024-09-13 14:22:42.885397222 +0000 UTC m=+0.362758724 container start 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T14:22:43.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:42 smithi017 podman[121389]: 2024-09-13 14:22:42.971109199 +0000 UTC m=+0.448470717 container attach 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T14:22:43.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 podman[121389]: 2024-09-13 14:22:43.08248932 +0000 UTC m=+0.559850827 container died 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:22:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:43 smithi189 ceph-mon[90992]: pgmap v144: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:43 smithi189 ceph-mon[90992]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:43 smithi189 ceph-mon[90992]: osdmap e89: 8 total, 7 up, 8 in 2024-09-13T14:22:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:43 smithi189 ceph-mon[90992]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[107556]: pgmap v144: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[107556]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[107556]: osdmap e89: 8 total, 7 up, 8 in 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[107556]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[103001]: pgmap v144: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[103001]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[103001]: osdmap e89: 8 total, 7 up, 8 in 2024-09-13T14:22:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:43 smithi017 ceph-mon[103001]: from='client.34317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:44.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 podman[121432]: 2024-09-13 14:22:43.750285866 +0000 UTC m=+0.661085628 container remove 96f9a1d229dc6935cafe09b996d5c836463adad44e565938111df4c831b78679 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T14:22:44.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T14:22:44.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 systemd[1]: Stopped Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:44.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service: Consumed 6.934s CPU time. 2024-09-13T14:22:44.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:43 smithi017 systemd[1]: Starting Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:44.479 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:22:44.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:44.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:22:44.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:44.480 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 podman[121533]: 2024-09-13 14:22:44.102041121 +0000 UTC m=+0.016411309 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:44.480 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 podman[121533]: 2024-09-13 14:22:44.226683686 +0000 UTC m=+0.141053915 container create 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T14:22:44.480 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 podman[121533]: 2024-09-13 14:22:44.479582256 +0000 UTC m=+0.393952462 container init 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 podman[121533]: 2024-09-13 14:22:44.486468795 +0000 UTC m=+0.400838984 container start 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 podman[121533]: 2024-09-13 14:22:44.579187811 +0000 UTC m=+0.493558014 container attach 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD) 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.734 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:22:44.735 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:22:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:44 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:22:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:44 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:44.927 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:45.023 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:22:45.023 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:44 smithi017 bash[121533]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:22:45.023 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:22:45.023 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 bash[121533]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:22:45.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-mon[103001]: pgmap v146: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:45.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/774360913' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:45.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-mon[107556]: pgmap v146: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:45.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/774360913' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 bash[121533]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 bash[121533]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 bash[121533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[121548]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:22:45.400 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 bash[121533]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:22:45.401 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 podman[121533]: 2024-09-13 14:22:45.052612591 +0000 UTC m=+0.966982787 container died 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:22:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:45 smithi189 ceph-mon[90992]: pgmap v146: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/774360913' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:45.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 podman[121754]: 2024-09-13 14:22:45.691380104 +0000 UTC m=+0.632465285 container remove 007a6e385cb9801417c0f8f81af14c5e4366bf82dacecd623db759dec5c1a202 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T14:22:45.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 podman[121824]: 2024-09-13 14:22:45.807865986 +0000 UTC m=+0.016611849 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:22:45.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:45 smithi017 podman[121824]: 2024-09-13 14:22:45.909015505 +0000 UTC m=+0.117761364 container create bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:22:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:46 smithi017 ceph-mon[103001]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:46 smithi017 ceph-mon[103001]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:46 smithi017 ceph-mon[107556]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:46 smithi017 ceph-mon[107556]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:46.451 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:46 smithi017 podman[121824]: 2024-09-13 14:22:46.118349824 +0000 UTC m=+0.327095694 container init bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:22:46.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:46 smithi017 podman[121824]: 2024-09-13 14:22:46.123650735 +0000 UTC m=+0.332396591 container start bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T14:22:46.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:46 smithi017 ceph-osd[121862]: -- 172.21.15.17:0/540540297 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b38ec81a40 con 0x55b38fa4e400 2024-09-13T14:22:46.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:46 smithi017 bash[121824]: bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 2024-09-13T14:22:46.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:46 smithi017 systemd[1]: Started Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:46 smithi189 ceph-mon[90992]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:46 smithi189 ceph-mon[90992]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:47] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.51.0" 2024-09-13T14:22:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[107556]: pgmap v147: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[103001]: pgmap v147: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:47.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:22:47.514+0000 7fb0ec05a640 -1 Falling back to public interface 2024-09-13T14:22:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:47 smithi189 ceph-mon[90992]: pgmap v147: 73 pgs: 25 active+undersized, 4 peering, 3 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-09-13T14:22:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:47.830 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:48.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3242811113' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:48.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3242811113' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3242811113' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:49.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[103001]: pgmap v148: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:49.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[103001]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[103001]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-09-13T14:22:49.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[107556]: pgmap v148: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:49.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[107556]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[107556]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:49 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-09-13T14:22:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:49 smithi189 ceph-mon[90992]: pgmap v148: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:49 smithi189 ceph-mon[90992]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:49 smithi189 ceph-mon[90992]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:49 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-09-13T14:22:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:22:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:22:50.750 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: pgmap v149: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1829243991' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: pgmap v149: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:51.019 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1829243991' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:51.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:50 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: pgmap v149: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1829243991' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:51.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:50 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:51.271 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:22:51.092+0000 7fb0ec05a640 -1 osd.1 87 log_to_monitors true 2024-09-13T14:22:51.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[103001]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:51.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[103001]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:51.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:22:51.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[107556]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:51.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[107556]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:51.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:51 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:22:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:51 smithi189 ceph-mon[90992]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:51 smithi189 ceph-mon[90992]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:51 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:22:52.313 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:22:52.173+0000 7fb0dfdfc640 -1 osd.1 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:22:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[103001]: pgmap v150: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:22:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[103001]: osdmap e90: 8 total, 7 up, 8 in 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[107556]: pgmap v150: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[107556]: osdmap e90: 8 total, 7 up, 8 in 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:22:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:52 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' 2024-09-13T14:22:53.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:52 smithi189 ceph-mon[90992]: pgmap v150: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:53.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:52 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:22:53.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:52 smithi189 ceph-mon[90992]: osdmap e90: 8 total, 7 up, 8 in 2024-09-13T14:22:53.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:52 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:22:53.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:52 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563]' entity='osd.1' 2024-09-13T14:22:53.662 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:54.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:22:53.992+0000 7f90e4868640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:22:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:22:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563] boot 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2050907571' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:22:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563] boot 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2050907571' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:54.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:22:54.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: osd.1 [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563] boot 2024-09-13T14:22:54.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2050907571' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:22:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: pgmap v153: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[107556]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: pgmap v153: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:22:55.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:22:55.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:22:55.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:54 smithi017 ceph-mon[103001]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:22:55.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: pgmap v153: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:22:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:54 smithi189 ceph-mon[90992]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:22:56.577 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:56 smithi017 ceph-mon[103001]: pgmap v155: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/414016141' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:56 smithi017 ceph-mon[107556]: pgmap v155: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/414016141' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:57.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:56 smithi189 ceph-mon[90992]: pgmap v155: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:22:57.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/414016141' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:22:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:22:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:22:57] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.51.0" 2024-09-13T14:22:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[103001]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[103001]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[107556]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[107556]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:58 smithi189 ceph-mon[90992]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:58 smithi189 ceph-mon[90992]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:22:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[103001]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded) 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[107556]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded) 2024-09-13T14:22:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:22:59 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:22:59.492 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:22:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:59 smithi189 ceph-mon[90992]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:59 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded) 2024-09-13T14:22:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:22:59 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:23:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2491592474' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:00.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2491592474' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2491592474' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:01.422 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:01 smithi189 ceph-mon[90992]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:01 smithi189 ceph-mon[90992]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:01.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:01 smithi189 ceph-mon[90992]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[103001]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[103001]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[103001]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[107556]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[107556]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:01 smithi017 ceph-mon[107556]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:02.384 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:02.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:02 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1964630989' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:02.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:02 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1964630989' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:02 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1964630989' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:03 smithi017 ceph-mon[103001]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:03 smithi017 ceph-mon[103001]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:03 smithi017 ceph-mon[107556]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:03 smithi017 ceph-mon[107556]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:03 smithi189 ceph-mon[90992]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:03 smithi189 ceph-mon[90992]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:04 smithi017 ceph-mon[103001]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:04 smithi017 ceph-mon[107556]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:04 smithi189 ceph-mon[90992]: from='client.34449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:05.270 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:05 smithi017 ceph-mon[107556]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1341373260' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:05 smithi017 ceph-mon[103001]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1341373260' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:05 smithi189 ceph-mon[90992]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1341373260' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:06 smithi017 ceph-mon[103001]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:06 smithi017 ceph-mon[107556]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:06 smithi189 ceph-mon[90992]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:07 smithi017 ceph-mon[107556]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:07 smithi017 ceph-mon[107556]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:07 smithi017 ceph-mon[103001]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:07 smithi017 ceph-mon[103001]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:07.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:07] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.51.0" 2024-09-13T14:23:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:07 smithi189 ceph-mon[90992]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:07.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:07 smithi189 ceph-mon[90992]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:08.160 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2382130376' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2382130376' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2382130376' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:09.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[103001]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:09.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[103001]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:09.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:23:09.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[107556]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:09.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[107556]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:09.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:23:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:09 smithi189 ceph-mon[90992]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:09.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:09 smithi189 ceph-mon[90992]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:09.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[103001]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[103001]: Upgrade: osd.2 is safe to restart 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[107556]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:23:10.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:10 smithi017 ceph-mon[107556]: Upgrade: osd.2 is safe to restart 2024-09-13T14:23:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:10 smithi189 ceph-mon[90992]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:10 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:23:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:10 smithi189 ceph-mon[90992]: Upgrade: osd.2 is safe to restart 2024-09-13T14:23:11.059 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[103001]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/568403129' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[107556]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/568403129' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:11 smithi189 ceph-mon[90992]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:23:11.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:11.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/568403129' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:12.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: Upgrade: Updating osd.2 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: Deploying daemon osd.2 on smithi017 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:12.410 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: Upgrade: Updating osd.2 2024-09-13T14:23:12.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: Deploying daemon osd.2 on smithi017 2024-09-13T14:23:12.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:12.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: Upgrade: Updating osd.2 2024-09-13T14:23:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: Deploying daemon osd.2 on smithi017 2024-09-13T14:23:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:12.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:13 smithi017 ceph-mon[103001]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:13 smithi017 ceph-mon[107556]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:13 smithi189 ceph-mon[90992]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.813 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:13 smithi017 systemd[1]: Stopping Ceph osd.2 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:13.969 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:14.149 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[55364]: 2024-09-13T14:23:13.810+0000 7f992aef0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:23:14.150 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[55364]: 2024-09-13T14:23:13.810+0000 7f992aef0640 -1 osd.2 92 *** Got signal Terminated *** 2024-09-13T14:23:14.150 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[55364]: 2024-09-13T14:23:13.810+0000 7f992aef0640 -1 osd.2 92 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:23:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2467961356' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:14 smithi017 ceph-mon[103001]: osd.2 marked itself down and dead 2024-09-13T14:23:14.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2467961356' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:14.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:14 smithi017 ceph-mon[107556]: osd.2 marked itself down and dead 2024-09-13T14:23:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2467961356' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:14 smithi189 ceph-mon[90992]: osd.2 marked itself down and dead 2024-09-13T14:23:14.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:14 smithi017 podman[126719]: 2024-09-13 14:23:14.659044848 +0000 UTC m=+0.960399602 container died 2e705cedcfaf0bf5dbf6bce5ce63db65ce7e7280a7d0e7ace49215d9ac8b348b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2) 2024-09-13T14:23:15.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[103001]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[103001]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:15.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[103001]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:23:15.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[103001]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[107556]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[107556]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:15.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[107556]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:23:15.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:15 smithi017 ceph-mon[107556]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.388 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126719]: 2024-09-13 14:23:15.152595044 +0000 UTC m=+1.453949797 container cleanup 2e705cedcfaf0bf5dbf6bce5ce63db65ce7e7280a7d0e7ace49215d9ac8b348b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:15.388 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 bash[126719]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:23:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:15 smithi189 ceph-mon[90992]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:15 smithi189 ceph-mon[90992]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:15 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:15 smithi189 ceph-mon[90992]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:23:15.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:15 smithi189 ceph-mon[90992]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:15.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126786]: 2024-09-13 14:23:15.386867453 +0000 UTC m=+0.722971503 container remove 2e705cedcfaf0bf5dbf6bce5ce63db65ce7e7280a7d0e7ace49215d9ac8b348b (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, RELEASE=quincy-bbdadfa, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:15.974 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126881]: 2024-09-13 14:23:15.606872583 +0000 UTC m=+0.016830323 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:15.974 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126881]: 2024-09-13 14:23:15.721445466 +0000 UTC m=+0.131403235 container create 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:23:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:16 smithi017 ceph-mon[107556]: osdmap e94: 8 total, 7 up, 8 in 2024-09-13T14:23:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:16 smithi017 ceph-mon[103001]: osdmap e94: 8 total, 7 up, 8 in 2024-09-13T14:23:16.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126881]: 2024-09-13 14:23:15.974023775 +0000 UTC m=+0.383981506 container init 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T14:23:16.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:15 smithi017 podman[126881]: 2024-09-13 14:23:15.980251268 +0000 UTC m=+0.390209008 container start 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:23:16.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:16 smithi017 podman[126881]: 2024-09-13 14:23:16.057169549 +0000 UTC m=+0.467127296 container attach 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:16.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:16 smithi017 podman[126881]: 2024-09-13 14:23:16.176069619 +0000 UTC m=+0.586027350 container died 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T14:23:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:16 smithi189 ceph-mon[90992]: osdmap e94: 8 total, 7 up, 8 in 2024-09-13T14:23:16.866 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:17.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-mon[103001]: pgmap v167: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:17.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2973448295' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:17.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-mon[107556]: pgmap v167: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:17.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2973448295' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:17.350 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[126924]: 2024-09-13 14:23:17.020077238 +0000 UTC m=+0.837989136 container remove 0390d290dab8ffb35dbe7c41baed1504068b0eee6c3e9175d324738846795dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:23:17.351 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T14:23:17.351 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 systemd[1]: Stopped Ceph osd.2 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:17.351 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.2.service: Consumed 6.000s CPU time. 2024-09-13T14:23:17.351 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 systemd[1]: Starting Ceph osd.2 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:17 smithi189 ceph-mon[90992]: pgmap v167: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2973448295' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:17.603 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:17] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.51.0" 2024-09-13T14:23:17.603 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[127103]: 2024-09-13 14:23:17.372984124 +0000 UTC m=+0.017220403 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:17.604 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[127103]: 2024-09-13 14:23:17.488058491 +0000 UTC m=+0.132294813 container create dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T14:23:17.855 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[127103]: 2024-09-13 14:23:17.715625411 +0000 UTC m=+0.359861687 container init dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:23:17.856 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[127103]: 2024-09-13 14:23:17.720937015 +0000 UTC m=+0.365173299 container start dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T14:23:17.856 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 podman[127103]: 2024-09-13 14:23:17.823589871 +0000 UTC m=+0.467826157 container attach dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux ) 2024-09-13T14:23:18.167 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.167 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.167 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.167 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.167 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:23:18.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:17 smithi017 bash[127103]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:23:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-mon[107556]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-mon[107556]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-mon[103001]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-mon[103001]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:18.452 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:23:18.452 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 bash[127103]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:23:18.452 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:23:18.452 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 bash[127103]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:23:18.452 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 bash[127103]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 bash[127103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate[127117]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 bash[127103]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:23:18.453 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 podman[127103]: 2024-09-13 14:23:18.274104536 +0000 UTC m=+0.918340813 container died dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:23:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:18 smithi189 ceph-mon[90992]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:18 smithi189 ceph-mon[90992]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:19.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:18 smithi017 podman[127254]: 2024-09-13 14:23:18.902870857 +0000 UTC m=+0.618113958 container remove dcc9f127cdb7854fd25bdc4ad6270d2c1c4e59c6b9c613d4456cdf9369d15937 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T14:23:19.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 podman[127317]: 2024-09-13 14:23:19.022808679 +0000 UTC m=+0.016579078 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:19.168 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 podman[127317]: 2024-09-13 14:23:19.129046182 +0000 UTC m=+0.122816619 container create f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T14:23:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:19 smithi017 ceph-mon[103001]: pgmap v168: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:19 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:19 smithi017 ceph-mon[107556]: pgmap v168: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:19 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:19.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 podman[127317]: 2024-09-13 14:23:19.364934795 +0000 UTC m=+0.358705202 container init f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:23:19.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 podman[127317]: 2024-09-13 14:23:19.370365021 +0000 UTC m=+0.364135419 container start f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:23:19.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 ceph-osd[127377]: -- 172.21.15.17:0/4256676932 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55add891ba40 con 0x55add96e8400 2024-09-13T14:23:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:19 smithi189 ceph-mon[90992]: pgmap v168: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:19 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:19.785 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 bash[127317]: f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 2024-09-13T14:23:19.785 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:19 smithi017 systemd[1]: Started Ceph osd.2 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:19.785 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/577188100' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/577188100' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/577188100' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:21.363 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T14:23:21.034+0000 7fdf08efd640 -1 Falling back to public interface 2024-09-13T14:23:21.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[103001]: pgmap v169: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:21.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[103001]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:21.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[103001]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:21.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[107556]: pgmap v169: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:21.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[107556]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:21.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:21 smithi017 ceph-mon[107556]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:21.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:21 smithi189 ceph-mon[90992]: pgmap v169: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:21.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:21 smithi189 ceph-mon[90992]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:21.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:21 smithi189 ceph-mon[90992]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:22.705 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:23.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:23 smithi017 ceph-mon[107556]: pgmap v170: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:23.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:23 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1361733272' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:23.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:23 smithi017 ceph-mon[103001]: pgmap v170: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:23.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:23 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1361733272' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:23 smithi189 ceph-mon[90992]: pgmap v170: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:23 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1361733272' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.539 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:24.539 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.539 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.539 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.539 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:24.818 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:24 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T14:23:24.599+0000 7fdf08efd640 -1 osd.2 92 log_to_monitors true 2024-09-13T14:23:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[103001]: pgmap v171: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[103001]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[103001]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[107556]: pgmap v171: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[107556]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-mon[107556]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:25 smithi189 ceph-mon[90992]: pgmap v171: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:25 smithi189 ceph-mon[90992]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:25 smithi189 ceph-mon[90992]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:23:25.619 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:25.949 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 14:23:25 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T14:23:25.711+0000 7fdefcc9f640 -1 osd.2 92 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2248806558' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: osdmap e95: 8 total, 7 up, 8 in 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[103001]: from='osd.2 ' entity='osd.2' 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2248806558' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: osdmap e95: 8 total, 7 up, 8 in 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:26 smithi017 ceph-mon[107556]: from='osd.2 ' entity='osd.2' 2024-09-13T14:23:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2248806558' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:23:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: osdmap e95: 8 total, 7 up, 8 in 2024-09-13T14:23:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: from='osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:26 smithi189 ceph-mon[90992]: from='osd.2 ' entity='osd.2' 2024-09-13T14:23:27.380 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:27] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.51.0" 2024-09-13T14:23:27.380 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.380 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: pgmap v173: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:27.380 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397] boot 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: pgmap v173: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:27.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397] boot 2024-09-13T14:23:27.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:23:27.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:23:27.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:27.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: from='client.34593 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: pgmap v173: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: osd.2 [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397] boot 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:23:27.546+0000 7f90e4868640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T14:23:28.530 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: pgmap v175: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3510304350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: pgmap v175: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:23:28.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3510304350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: pgmap v175: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:23:28.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3510304350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[103001]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[103001]: from='client.44498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[107556]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:29 smithi017 ceph-mon[107556]: from='client.44498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:29 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2024-09-13T14:23:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:29 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:23:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:29 smithi189 ceph-mon[90992]: from='client.34611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:29.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:29 smithi189 ceph-mon[90992]: from='client.44498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:30 smithi189 ceph-mon[90992]: pgmap v177: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:30 smithi017 ceph-mon[103001]: pgmap v177: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:30 smithi017 ceph-mon[107556]: pgmap v177: 73 pgs: 16 peering, 3 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:31.417 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/781212792' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/781212792' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/781212792' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[107556]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[107556]: pgmap v178: 73 pgs: 16 peering, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[107556]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[103001]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[103001]: pgmap v178: 73 pgs: 16 peering, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:32 smithi017 ceph-mon[103001]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:32 smithi189 ceph-mon[90992]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:32 smithi189 ceph-mon[90992]: pgmap v178: 73 pgs: 16 peering, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:32 smithi189 ceph-mon[90992]: from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:34.310 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[107556]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2271553302' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[107556]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[103001]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2271553302' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:34 smithi017 ceph-mon[103001]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:35.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:34 smithi189 ceph-mon[90992]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:35.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2271553302' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:35.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:34 smithi189 ceph-mon[90992]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:35 smithi017 ceph-mon[103001]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:35 smithi017 ceph-mon[107556]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:35 smithi189 ceph-mon[90992]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:37.200 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:36 smithi017 ceph-mon[107556]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:36 smithi017 ceph-mon[103001]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:36 smithi189 ceph-mon[90992]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:37] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.51.0" 2024-09-13T14:23:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/219715341' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:37 smithi017 ceph-mon[103001]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/219715341' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:37 smithi017 ceph-mon[107556]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/219715341' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:37 smithi189 ceph-mon[90992]: from='client.34665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:38 smithi017 ceph-mon[103001]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:38 smithi017 ceph-mon[103001]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:38 smithi017 ceph-mon[107556]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:38 smithi017 ceph-mon[107556]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:38 smithi189 ceph-mon[90992]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:38 smithi189 ceph-mon[90992]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:40.085 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[103001]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3792224959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[103001]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[103001]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[107556]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3792224959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[107556]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:40 smithi017 ceph-mon[107556]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:40 smithi189 ceph-mon[90992]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:41.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3792224959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:41.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:40 smithi189 ceph-mon[90992]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:41.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:40 smithi189 ceph-mon[90992]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:42.987 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1721170712' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1721170712' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:23:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1721170712' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: Upgrade: osd.3 is safe to restart 2024-09-13T14:23:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: Upgrade: osd.3 is safe to restart 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:44.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:23:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: Upgrade: osd.3 is safe to restart 2024-09-13T14:23:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:44.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:44.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:23:44.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:23:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[103001]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[103001]: Upgrade: Updating osd.3 2024-09-13T14:23:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[103001]: Deploying daemon osd.3 on smithi017 2024-09-13T14:23:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[107556]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[107556]: Upgrade: Updating osd.3 2024-09-13T14:23:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:45 smithi017 ceph-mon[107556]: Deploying daemon osd.3 on smithi017 2024-09-13T14:23:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:45 smithi189 ceph-mon[90992]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:45 smithi189 ceph-mon[90992]: Upgrade: Updating osd.3 2024-09-13T14:23:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:45 smithi189 ceph-mon[90992]: Deploying daemon osd.3 on smithi017 2024-09-13T14:23:45.899 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:46 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3925508414' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:46 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3925508414' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:46.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:46 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3925508414' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:47.151 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:46 smithi017 systemd[1]: Stopping Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:47.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[103001]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:47.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[103001]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:47.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[103001]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:47.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[107556]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:47.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[107556]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:47.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-mon[107556]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:47] "GET /metrics HTTP/1.1" 200 34103 "" "Prometheus/2.51.0" 2024-09-13T14:23:47.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[61509]: 2024-09-13T14:23:47.149+0000 7f37dae1e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:23:47.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[61509]: 2024-09-13T14:23:47.149+0000 7f37dae1e640 -1 osd.3 97 *** Got signal Terminated *** 2024-09-13T14:23:47.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[61509]: 2024-09-13T14:23:47.149+0000 7f37dae1e640 -1 osd.3 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:23:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:47 smithi189 ceph-mon[90992]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:47 smithi189 ceph-mon[90992]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:47 smithi189 ceph-mon[90992]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:48 smithi017 ceph-mon[103001]: osd.3 marked itself down and dead 2024-09-13T14:23:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:48 smithi017 ceph-mon[107556]: osd.3 marked itself down and dead 2024-09-13T14:23:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:48 smithi189 ceph-mon[90992]: osd.3 marked itself down and dead 2024-09-13T14:23:48.792 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:48.924 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:48 smithi017 podman[132713]: 2024-09-13 14:23:48.629557458 +0000 UTC m=+1.592439180 container died 81c4e9274ff0620a1e56ab70cbfb2481b59d73dbbd9b36db6453733b350e49f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T14:23:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[107556]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[107556]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:23:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1686690319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:49.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132713]: 2024-09-13 14:23:49.022394787 +0000 UTC m=+1.985276506 container cleanup 81c4e9274ff0620a1e56ab70cbfb2481b59d73dbbd9b36db6453733b350e49f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T14:23:49.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 bash[132713]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:23:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[103001]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[103001]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:23:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1686690319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:49 smithi189 ceph-mon[90992]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:49 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:23:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:49 smithi189 ceph-mon[90992]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T14:23:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1686690319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:49.641 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132780]: 2024-09-13 14:23:49.207497432 +0000 UTC m=+0.576494167 container remove 81c4e9274ff0620a1e56ab70cbfb2481b59d73dbbd9b36db6453733b350e49f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T14:23:49.928 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132898]: 2024-09-13 14:23:49.544120281 +0000 UTC m=+0.016479546 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:49.928 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132898]: 2024-09-13 14:23:49.692270892 +0000 UTC m=+0.164630209 container create de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T14:23:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[107556]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[107556]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[107556]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T14:23:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[103001]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[103001]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:50 smithi017 ceph-mon[103001]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T14:23:50.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132898]: 2024-09-13 14:23:49.928162121 +0000 UTC m=+0.400521385 container init de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T14:23:50.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:49 smithi017 podman[132898]: 2024-09-13 14:23:49.933108365 +0000 UTC m=+0.405467630 container start de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True) 2024-09-13T14:23:50.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 podman[132898]: 2024-09-13 14:23:50.011161576 +0000 UTC m=+0.483520841 container attach de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:50.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 podman[132898]: 2024-09-13 14:23:50.123844296 +0000 UTC m=+0.596203571 container died de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:23:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:50 smithi189 ceph-mon[90992]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:50 smithi189 ceph-mon[90992]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:50 smithi189 ceph-mon[90992]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T14:23:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:23:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:23:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:23:51.109 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 podman[132942]: 2024-09-13 14:23:50.781989062 +0000 UTC m=+0.652340734 container remove de2c1d0d74ebf3060cf5b75e10382f724adf76fee6f47365b23c4d007444b03f (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T14:23:51.110 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:23:51.110 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 systemd[1]: Stopped Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:51.110 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:50 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service: Consumed 5.325s CPU time. 2024-09-13T14:23:51.110 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 systemd[1]: Starting Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:23:51.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-mon[103001]: pgmap v189: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:51.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-mon[107556]: pgmap v189: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:51.367 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 podman[133046]: 2024-09-13 14:23:51.136855007 +0000 UTC m=+0.023127076 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:51.367 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 podman[133046]: 2024-09-13 14:23:51.24983167 +0000 UTC m=+0.136103724 container create 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T14:23:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:51 smithi189 ceph-mon[90992]: pgmap v189: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:51.711 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 podman[133046]: 2024-09-13 14:23:51.46113531 +0000 UTC m=+0.347407373 container init 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 podman[133046]: 2024-09-13 14:23:51.46703974 +0000 UTC m=+0.353311801 container start 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 podman[133046]: 2024-09-13 14:23:51.560581638 +0000 UTC m=+0.446853713 container attach 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:51.727 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:23:51.728 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:23:52.031 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:23:52.032 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:51 smithi017 bash[133046]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:23:52.032 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:23:52.032 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 bash[133046]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:23:52.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3727737676' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:52.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3727737676' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:52.399 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:23:52.399 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 bash[133046]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:23:52.399 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:23:52.399 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 bash[133046]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:23:52.400 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:23:52.400 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 bash[133046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:23:52.400 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate[133079]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:23:52.400 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 bash[133046]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:23:52.400 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 podman[133046]: 2024-09-13 14:23:52.060949641 +0000 UTC m=+0.947221714 container died 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:23:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3727737676' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:52.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 podman[133266]: 2024-09-13 14:23:52.681286405 +0000 UTC m=+0.614382801 container remove 7de1885575dbd6c21c400a3420cb2a25de04de3195a29b5d1e06de696ffde725 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:23:52.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 podman[133333]: 2024-09-13 14:23:52.890782753 +0000 UTC m=+0.109012275 container create 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-09-13T14:23:52.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:52 smithi017 podman[133333]: 2024-09-13 14:23:52.79836789 +0000 UTC m=+0.016597413 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[103001]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[103001]: pgmap v190: 73 pgs: 3 active+undersized, 12 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[103001]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[107556]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[107556]: pgmap v190: 73 pgs: 3 active+undersized, 12 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-mon[107556]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:53.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:53 smithi017 podman[133333]: 2024-09-13 14:23:53.151993966 +0000 UTC m=+0.370223488 container init 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T14:23:53.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:53 smithi017 podman[133333]: 2024-09-13 14:23:53.157036861 +0000 UTC m=+0.375266384 container start 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T14:23:53.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:53 smithi017 ceph-osd[133371]: -- 172.21.15.17:0/1013977099 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561add9dba40 con 0x561ade7a8000 2024-09-13T14:23:53.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:53 smithi017 bash[133333]: 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 2024-09-13T14:23:53.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:53 smithi017 systemd[1]: Started Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:23:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:53 smithi189 ceph-mon[90992]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:53 smithi189 ceph-mon[90992]: pgmap v190: 73 pgs: 3 active+undersized, 12 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:23:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:53 smithi189 ceph-mon[90992]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:54.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/674458405' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:54.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:54.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/674458405' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:54.647 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:23:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/674458405' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:55.033 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:54 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T14:23:54.807+0000 7fef66c8d640 -1 Falling back to public interface 2024-09-13T14:23:55.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:55 smithi189 ceph-mon[90992]: pgmap v191: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:55.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:55 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:55 smithi017 ceph-mon[103001]: pgmap v191: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:55 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:55 smithi017 ceph-mon[107556]: pgmap v191: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:55 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:23:56.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:56 smithi017 ceph-mon[103001]: from='client.44657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:56.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:56 smithi017 ceph-mon[103001]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:56.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:56 smithi017 ceph-mon[107556]: from='client.44657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:56.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:56 smithi017 ceph-mon[107556]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:56 smithi189 ceph-mon[90992]: from='client.44657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:56 smithi189 ceph-mon[90992]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:57.577 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-13T14:23:57.577 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:23:57] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.51.0" 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: pgmap v192: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2180413867' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:57.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: pgmap v192: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:57.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2180413867' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: pgmap v192: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:23:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2180413867' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:23:58.658 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T14:23:58.359+0000 7fef66c8d640 -1 osd.3 97 log_to_monitors true 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: pgmap v193: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[103001]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:23:58.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: pgmap v193: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-mon[107556]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:23:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: pgmap v193: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='client.34800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:58 smithi189 ceph-mon[90992]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:23:58.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 14:23:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T14:23:58.657+0000 7fef5ea37640 -1 osd.3 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:23:59.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[103001]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:23:59.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[103001]: osdmap e100: 8 total, 7 up, 8 in 2024-09-13T14:23:59.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[103001]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:23:59.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[107556]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:23:59.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[107556]: osdmap e100: 8 total, 7 up, 8 in 2024-09-13T14:23:59.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:23:59 smithi017 ceph-mon[107556]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:24:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:59 smithi189 ceph-mon[90992]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:24:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:59 smithi189 ceph-mon[90992]: osdmap e100: 8 total, 7 up, 8 in 2024-09-13T14:24:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:23:59 smithi189 ceph-mon[90992]: from='osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:24:00.229 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:24:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:00.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:00.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290] boot 2024-09-13T14:24:00.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:24:00.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:24:00.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: pgmap v196: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:00.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/640702514' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:24:00.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:00 smithi189 ceph-mon[90992]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:24:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290] boot 2024-09-13T14:24:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:24:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:24:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: pgmap v196: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/640702514' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[103001]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: osd.3 [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290] boot 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: pgmap v196: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/640702514' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:24:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:00 smithi017 ceph-mon[107556]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:24:01.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:24:01.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:01.621 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:01 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T14:24:01.533+0000 7f90e4868640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[103001]: pgmap v198: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:24:02.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:02 smithi017 ceph-mon[107556]: pgmap v198: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:24:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:02 smithi189 ceph-mon[90992]: pgmap v198: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-09-13T14:24:03.373 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:04.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:24:04.314 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[107556]: from='client.44696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[107556]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:04.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[103001]: from='client.44696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[103001]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:04.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:04 smithi189 ceph-mon[90992]: from='client.44696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:04 smithi189 ceph-mon[90992]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[103001]: pgmap v199: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:24:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:24:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[107556]: pgmap v199: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:24:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:05 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:24:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:05 smithi189 ceph-mon[90992]: pgmap v199: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:05 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-09-13T14:24:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:05 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:24:06.235 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:24:06.235 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) 9s ago 11m 18.6M - 0.27.0 11f11916f8cd ceb612349939 2024-09-13T14:24:06.235 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (6m) 8s ago 11m 85.6M - 9.4.7 954c08fa6188 d0c012ef79a6 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (10m) 9s ago 10m 18.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (10m) 8s ago 10m 15.0M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443,9283,8765 running (6m) 8s ago 15m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 e2be48bfd3a8 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) 9s ago 16m 518M - 19.3.0-4697-gd58f7306 fec07016a2b0 b59080328aea 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (6m) 9s ago 17m 48.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c2de9b169292 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (4m) 8s ago 15m 39.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 37d58338d569 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4m) 9s ago 15m 41.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c1ddeb604cac 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (7m) 9s ago 11m 11.9M - 1.7.0 72c9c2088986 fce992870ef4 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (6m) 8s ago 10m 12.7M - 1.7.0 72c9c2088986 68ff3e2b1910 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (2m) 9s ago 15m 49.8M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 b6c1265d9453 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (80s) 9s ago 14m 48.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 bc4e38b82905 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (47s) 9s ago 14m 45.9M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 f910c1d6db6a 2024-09-13T14:24:06.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (13s) 9s ago 13m 14.4M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 44369002d276 2024-09-13T14:24:06.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (13m) 8s ago 13m 51.3M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:24:06.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (13m) 8s ago 13m 48.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:24:06.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (12m) 8s ago 12m 48.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:24:06.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (12m) 8s ago 12m 56.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:24:06.237 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (6m) 8s ago 11m 54.2M - 2.51.0 1d3b7f56885b 770a1c26888c 2024-09-13T14:24:06.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:24:07.113 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:07.496 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:07 smithi017 ceph-mon[103001]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:07.496 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:07 smithi017 ceph-mon[103001]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:07.496 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:07] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.51.0" 2024-09-13T14:24:07.497 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:07 smithi017 ceph-mon[107556]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:07.497 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:07 smithi017 ceph-mon[107556]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:07 smithi189 ceph-mon[90992]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:07 smithi189 ceph-mon[90992]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:08.887 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:24:08.887 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) 11s ago 11m 18.6M - 0.27.0 11f11916f8cd ceb612349939 2024-09-13T14:24:08.887 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi189 *:3000 running (6m) 11s ago 11m 85.6M - 9.4.7 954c08fa6188 d0c012ef79a6 2024-09-13T14:24:08.887 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ldrmqj smithi017 running (10m) 11s ago 10m 18.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 64fc4881d739 2024-09-13T14:24:08.887 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi189.ndtccr smithi189 running (10m) 11s ago 10m 15.0M - 17.2.7-1627-gbbdadfa3 a994b00deb3a ea59bbe270db 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi189 *:8443,9283,8765 running (6m) 11s ago 15m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 e2be48bfd3a8 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) 11s ago 16m 518M - 19.3.0-4697-gd58f7306 fec07016a2b0 b59080328aea 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (6m) 11s ago 17m 48.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c2de9b169292 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi189 running (4m) 11s ago 15m 39.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 37d58338d569 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4m) 11s ago 15m 41.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 c1ddeb604cac 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (7m) 11s ago 11m 11.9M - 1.7.0 72c9c2088986 fce992870ef4 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi189 *:9100 running (6m) 11s ago 10m 12.7M - 1.7.0 72c9c2088986 68ff3e2b1910 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (2m) 11s ago 15m 49.8M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 b6c1265d9453 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (83s) 11s ago 14m 48.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 bc4e38b82905 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (49s) 11s ago 14m 45.9M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 f910c1d6db6a 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (16s) 11s ago 13m 14.4M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 44369002d276 2024-09-13T14:24:08.888 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi189 running (13m) 11s ago 13m 51.3M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a a06c4fb486c5 2024-09-13T14:24:08.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi189 running (13m) 11s ago 13m 48.8M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 768ac2ae3f92 2024-09-13T14:24:08.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi189 running (12m) 11s ago 12m 48.7M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a e712d52d9e99 2024-09-13T14:24:08.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi189 running (12m) 11s ago 12m 56.1M 2480M 17.2.7-1627-gbbdadfa3 a994b00deb3a 35088803cdca 2024-09-13T14:24:08.889 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi189 *:9095 running (6m) 11s ago 11m 54.2M - 2.51.0 1d3b7f56885b 770a1c26888c 2024-09-13T14:24:09.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:09 smithi017 ceph-mon[103001]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:09.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:09 smithi017 ceph-mon[107556]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:09.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions' 2024-09-13T14:24:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:09 smithi189 ceph-mon[90992]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:09.798 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:10 smithi017 ceph-mon[103001]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:10 smithi017 ceph-mon[107556]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:10 smithi189 ceph-mon[90992]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:11 smithi189 ceph-mon[90992]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:11 smithi017 ceph-mon[103001]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:11 smithi017 ceph-mon[107556]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: "mon": { 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr": { 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:24:11.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "osd": { 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 4, 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "mds": { 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "overall": { 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 6, 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 9 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:24:11.702 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:24:12.381 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:12.381 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:24:12.391 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-13T14:24:12.391 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:24:12.414 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:24:12.414 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:24:12.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:24:12.672 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/350576519' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/350576519' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/350576519' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:24:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:13 smithi017 ceph-mon[103001]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:13 smithi017 ceph-mon[107556]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:13 smithi189 ceph-mon[90992]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:14.675 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:24:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:14 smithi017 ceph-mon[103001]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:14 smithi017 ceph-mon[107556]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:14 smithi189 ceph-mon[90992]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:15.280 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":40,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":126252659},{"type":"v1","addr":"172.21.15.17:6801","nonce":126252659}]},"active_addr":"172.21.15.17:6801/126252659","active_change":"2024-09-13T14:18:11.615036+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":26110,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1563823054}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1065420260}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1224462774}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1328535979}]}]} 2024-09-13T14:24:15.285 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:24:15.285 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:24:15.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:24:15.543 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:15.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1709282161' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:24:15.665 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1709282161' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:24:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1709282161' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:24:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:16 smithi017 ceph-mon[103001]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:16 smithi017 ceph-mon[107556]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:16 smithi189 ceph-mon[90992]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:17.483 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:24:17.483 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":102,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:24:00.637335+0000","last_up_change":"2024-09-13T14:23:59.635717+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":101,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6803","nonce":59727078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6805","nonce":59727078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6809","nonce":59727078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6807","nonce":59727078}]},"public_addr":"172.21.15.17:6803/59727078","cluster_addr":"172.21.15.17:6805/59727078","heartbeat_back_addr":"172.21.15.17:6809/59727078","heartbeat_front_addr":"172.21.15.17:6807/59727078","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":87,"up_from":91,"up_thru":101,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6811","nonce":3641293563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6813","nonce":3641293563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6817","nonce":3641293563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6815","nonce":3641293563}]},"public_addr":"172.21.15.17:6811/3641293563","cluster_addr":"172.21.15.17:6813/3641293563","heartbeat_back_addr":"172.21.15.17:6817/3641293563","heartbeat_front_addr":"172.21.15.17:6815/3641293563","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":92,"up_from":96,"up_thru":101,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6819","nonce":847801397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6821","nonce":847801397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6825","nonce":847801397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6823","nonce":847801397}]},"public_addr":"172.21.15.17:6819/847801397","cluster_addr":"172.21.15.17:6821/847801397","heartbeat_back_addr":"172.21.15.17:6825/847801397","heartbeat_front_addr":"172.21.15.17:6823/847801397","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":97,"up_from":101,"up_thru":101,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6827","nonce":594040290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6829","nonce":594040290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6833","nonce":594040290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6831","nonce":594040290}]},"public_addr":"172.21.15.17:6827/594040290","cluster_addr":"172.21.15.17:6829/594040290","heartbeat_back_addr":"172.21.15.17:6833/594040290","heartbeat_front_addr":"172.21.15.17:6831/594040290","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":87},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":92},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":97},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6801/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3929495183":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3205338252":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/1432283173":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.189:6835/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/2755559292":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3987946356":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/3334536375":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/2573022556":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/652586513":"2024-09-14T14:17:29.088581+0000","172.21.15.189:0/2946870083":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3476899836":"2024-09-14T14:16:12.599650+0000","172.21.15.189:6834/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/1507399137":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/282666625":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1515064414":"2024-09-14T14:16:12.599650+0000","172.21.15.189:0/4174048439":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/101070012":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.189:0/2620105210":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:24:17.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:17] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.51.0" 2024-09-13T14:24:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:24:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:24:18.145 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:24:18.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:24:18.402 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2250719972' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:18 smithi189 ceph-mon[90992]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:18.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2250719972' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:18.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:18 smithi017 ceph-mon[103001]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:18.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2250719972' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:18.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:18 smithi017 ceph-mon[107556]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:20.402 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:24:20.402 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":102,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T14:24:00.637335+0000","last_up_change":"2024-09-13T14:23:59.635717+0000","last_in_change":"2024-09-13T14:11:36.805213+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":101,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6803","nonce":59727078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6805","nonce":59727078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6809","nonce":59727078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6807","nonce":59727078}]},"public_addr":"172.21.15.17:6803/59727078","cluster_addr":"172.21.15.17:6805/59727078","heartbeat_back_addr":"172.21.15.17:6809/59727078","heartbeat_front_addr":"172.21.15.17:6807/59727078","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":87,"up_from":91,"up_thru":101,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6811","nonce":3641293563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6813","nonce":3641293563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6817","nonce":3641293563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":3641293563},{"type":"v1","addr":"172.21.15.17:6815","nonce":3641293563}]},"public_addr":"172.21.15.17:6811/3641293563","cluster_addr":"172.21.15.17:6813/3641293563","heartbeat_back_addr":"172.21.15.17:6817/3641293563","heartbeat_front_addr":"172.21.15.17:6815/3641293563","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":92,"up_from":96,"up_thru":101,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6819","nonce":847801397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6821","nonce":847801397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6825","nonce":847801397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6823","nonce":847801397}]},"public_addr":"172.21.15.17:6819/847801397","cluster_addr":"172.21.15.17:6821/847801397","heartbeat_back_addr":"172.21.15.17:6825/847801397","heartbeat_front_addr":"172.21.15.17:6823/847801397","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":97,"up_from":101,"up_thru":101,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6827","nonce":594040290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6829","nonce":594040290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6833","nonce":594040290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6831","nonce":594040290}]},"public_addr":"172.21.15.17:6827/594040290","cluster_addr":"172.21.15.17:6829/594040290","heartbeat_back_addr":"172.21.15.17:6833/594040290","heartbeat_front_addr":"172.21.15.17:6831/594040290","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6809","nonce":379852919}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6811","nonce":379852919}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6815","nonce":379852919}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":379852919},{"type":"v1","addr":"172.21.15.189:6813","nonce":379852919}]},"public_addr":"172.21.15.189:6809/379852919","cluster_addr":"172.21.15.189:6811/379852919","heartbeat_back_addr":"172.21.15.189:6815/379852919","heartbeat_front_addr":"172.21.15.189:6813/379852919","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":98,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":87},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":92},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":97},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6801/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3929495183":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3205338252":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/1432283173":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.189:6835/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/2755559292":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3987946356":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/3334536375":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/2573022556":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/652586513":"2024-09-14T14:17:29.088581+0000","172.21.15.189:0/2946870083":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3476899836":"2024-09-14T14:16:12.599650+0000","172.21.15.189:6834/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/1507399137":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:6801/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/282666625":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1515064414":"2024-09-14T14:16:12.599650+0000","172.21.15.189:0/4174048439":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/101070012":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.189:0/2620105210":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:24:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:21.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:20 smithi017 ceph-mon[103001]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:21.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3137157673' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:21.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:20 smithi017 ceph-mon[107556]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:21.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3137157673' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:21.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:24:21.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:24:21.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:24:21.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:24:21.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:24:21.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:24:21.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:24:21.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:24:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:20 smithi189 ceph-mon[90992]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3137157673' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:24:21.811 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.815 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.815 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.816 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.816 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.822 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.823 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:21.826 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:22 smithi017 ceph-mon[103001]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:22 smithi017 ceph-mon[107556]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:23.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:22 smithi189 ceph-mon[90992]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:25.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:25 smithi017 ceph-mon[107556]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:25.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:25 smithi017 ceph-mon[103001]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:25 smithi189 ceph-mon[90992]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:27 smithi017 ceph-mon[103001]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:27.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:27] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.51.0" 2024-09-13T14:24:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:27 smithi017 ceph-mon[107556]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:27.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:27 smithi189 ceph-mon[90992]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:29 smithi017 ceph-mon[103001]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:29 smithi017 ceph-mon[107556]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:29 smithi189 ceph-mon[90992]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:31 smithi017 ceph-mon[103001]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:31 smithi017 ceph-mon[107556]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:31 smithi189 ceph-mon[90992]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:32.974 INFO:teuthology.orchestra.run.smithi017.stdout:184683593888 2024-09-13T14:24:32.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:24:33.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:33 smithi017 ceph-mon[103001]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:33.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:33 smithi017 ceph-mon[107556]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:33 smithi189 ceph-mon[90992]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:33.976 INFO:teuthology.orchestra.run.smithi017.stdout:390842023958 2024-09-13T14:24:33.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:24:34.664 INFO:teuthology.orchestra.run.smithi017.stdout:433791696904 2024-09-13T14:24:34.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:24:35.459 INFO:teuthology.orchestra.run.smithi017.stdout:158913790118 2024-09-13T14:24:35.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:24:35.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:35 smithi017 ceph-mon[103001]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:35.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:35 smithi017 ceph-mon[107556]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:35 smithi189 ceph-mon[90992]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:36.252 INFO:teuthology.orchestra.run.smithi017.stdout:133143986347 2024-09-13T14:24:36.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:24:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:37 smithi017 ceph-mon[103001]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:37] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.51.0" 2024-09-13T14:24:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:37 smithi017 ceph-mon[107556]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:37 smithi189 ceph-mon[90992]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:38.037 INFO:teuthology.orchestra.run.smithi017.stdout:210453397661 2024-09-13T14:24:38.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:24:38.512 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:38.512 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:39.227 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:39.236 INFO:teuthology.orchestra.run.smithi017.stdout:369367187493 2024-09-13T14:24:39.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:24:39.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:39 smithi017 ceph-mon[103001]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:39.483 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:39 smithi017 ceph-mon[107556]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:39 smithi189 ceph-mon[90992]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:39.897 INFO:teuthology.orchestra.run.smithi017.stdout:412316860432 2024-09-13T14:24:39.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:24:40.240 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:41 smithi017 ceph-mon[103001]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:41 smithi017 ceph-mon[107556]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:41 smithi189 ceph-mon[90992]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:42.178 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:42.178 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:42.178 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:42.222 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:42.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:42.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:43.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:43 smithi017 ceph-mon[103001]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:43.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:43 smithi017 ceph-mon[107556]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:43 smithi189 ceph-mon[90992]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:43.900 INFO:teuthology.orchestra.run.smithi017.stdout:433791696906 2024-09-13T14:24:44.428 INFO:teuthology.orchestra.run.smithi017.stdout:158913790121 2024-09-13T14:24:44.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/289465386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:24:44.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/289465386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:24:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/289465386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:24:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:45 smithi017 ceph-mon[103001]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2114191111' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:24:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:45 smithi017 ceph-mon[107556]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2114191111' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:24:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:45 smithi189 ceph-mon[90992]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2114191111' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:24:47.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:47 smithi017 ceph-mon[103001]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:47.685 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:47] "GET /metrics HTTP/1.1" 200 34098 "" "Prometheus/2.51.0" 2024-09-13T14:24:47.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:47 smithi017 ceph-mon[107556]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:47.717 INFO:tasks.cephadm.ceph_manager.ceph:need seq 433791696904 got 433791696906 for osd.3 2024-09-13T14:24:47.718 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:47 smithi189 ceph-mon[90992]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:47.970 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790118 got 158913790121 for osd.5 2024-09-13T14:24:47.971 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:49.361 INFO:teuthology.orchestra.run.smithi017.stdout:133143986350 2024-09-13T14:24:49.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:49 smithi017 ceph-mon[103001]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:49.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:49 smithi017 ceph-mon[107556]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:49 smithi189 ceph-mon[90992]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:50.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4002188951' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:24:50.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4002188951' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:24:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4002188951' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:24:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:24:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:24:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:24:50.649 INFO:teuthology.orchestra.run.smithi017.stdout:369367187496 2024-09-13T14:24:51.373 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367187493 got 369367187496 for osd.0 2024-09-13T14:24:51.374 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:51 smithi017 ceph-mon[107556]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3429577954' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:24:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:51 smithi017 ceph-mon[103001]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3429577954' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:24:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:51 smithi189 ceph-mon[90992]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3429577954' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:24:52.214 INFO:teuthology.orchestra.run.smithi017.stdout:210453397664 2024-09-13T14:24:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/884522802' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:24:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/884522802' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:24:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/884522802' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:24:53.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:53 smithi017 ceph-mon[103001]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:53.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:53 smithi017 ceph-mon[107556]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:53 smithi189 ceph-mon[90992]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:53.959 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397661 got 210453397664 for osd.7 2024-09-13T14:24:53.960 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:54.437 INFO:teuthology.orchestra.run.smithi017.stdout:390842023962 2024-09-13T14:24:54.643 INFO:teuthology.orchestra.run.smithi017.stdout:184683593893 2024-09-13T14:24:55.261 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593888 got 184683593893 for osd.6 2024-09-13T14:24:55.261 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[107556]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/566384696' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4043733437' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[103001]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/566384696' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:24:55.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4043733437' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:24:55.613 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986347 got 133143986350 for osd.4 2024-09-13T14:24:55.613 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:55 smithi189 ceph-mon[90992]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:55.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/566384696' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:24:55.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4043733437' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:24:55.984 INFO:teuthology.orchestra.run.smithi017.stdout:412316860435 2024-09-13T14:24:56.201 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842023958 got 390842023962 for osd.1 2024-09-13T14:24:56.201 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3642997069' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:24:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3642997069' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:24:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3642997069' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:24:56.765 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860432 got 412316860435 for osd.2 2024-09-13T14:24:56.765 DEBUG:teuthology.parallel:result is None 2024-09-13T14:24:56.765 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:24:56.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:24:57.028 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:57 smithi017 ceph-mon[103001]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:57.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:24:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:24:57] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.51.0" 2024-09-13T14:24:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:57 smithi017 ceph-mon[107556]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:57 smithi189 ceph-mon[90992]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:24:58.899 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:24:58.899 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:24:59.478 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:24:59 smithi017 ceph-mon[103001]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:59.478 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:24:59 smithi017 ceph-mon[107556]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:59.504 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":226,"stamp":"2024-09-13T14:24:57.723089+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338996,"kb_used_data":22820,"kb_used_omap":35,"kb_used_meta":1315996,"kb_avail":748622220,"statfs":{"total":767960285184,"available":766589153280,"internally_reserved":0,"allocated":23367680,"data_stored":19051247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1347580921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002993"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.661336+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:24:00.661336+0000","last_peered":"2024-09-13T14:24:00.661336+0000","last_clean":"2024-09-13T14:24:00.661336+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:24:00.661336+0000","last_undegraded":"2024-09-13T14:24:00.661336+0000","last_fullsized":"2024-09-13T14:24:00.661336+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644047+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:23:59.644047+0000","last_peered":"2024-09-13T14:23:59.644047+0000","last_clean":"2024-09-13T14:23:59.644047+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:23:59.644047+0000","last_undegraded":"2024-09-13T14:23:59.644047+0000","last_fullsized":"2024-09-13T14:23:59.644047+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653974+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:24:00.653974+0000","last_peered":"2024-09-13T14:24:00.653974+0000","last_clean":"2024-09-13T14:24:00.653974+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:24:00.653974+0000","last_undegraded":"2024-09-13T14:24:00.653974+0000","last_fullsized":"2024-09-13T14:24:00.653974+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655023+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:24:00.655023+0000","last_peered":"2024-09-13T14:24:00.655023+0000","last_clean":"2024-09-13T14:24:00.655023+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:24:00.655023+0000","last_undegraded":"2024-09-13T14:24:00.655023+0000","last_fullsized":"2024-09-13T14:24:00.655023+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657045+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:24:00.657045+0000","last_peered":"2024-09-13T14:24:00.657045+0000","last_clean":"2024-09-13T14:24:00.657045+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:24:00.657045+0000","last_undegraded":"2024-09-13T14:24:00.657045+0000","last_fullsized":"2024-09-13T14:24:00.657045+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643390+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:23:59.643390+0000","last_peered":"2024-09-13T14:23:59.643390+0000","last_clean":"2024-09-13T14:23:59.643390+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:23:59.643390+0000","last_undegraded":"2024-09-13T14:23:59.643390+0000","last_fullsized":"2024-09-13T14:23:59.643390+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.651402+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:24:00.651402+0000","last_peered":"2024-09-13T14:24:00.651402+0000","last_clean":"2024-09-13T14:24:00.651402+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:24:00.651402+0000","last_undegraded":"2024-09-13T14:24:00.651402+0000","last_fullsized":"2024-09-13T14:24:00.651402+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":74,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639203+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:23:59.639203+0000","last_peered":"2024-09-13T14:23:59.639203+0000","last_clean":"2024-09-13T14:23:59.639203+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:23:59.639203+0000","last_undegraded":"2024-09-13T14:23:59.639203+0000","last_fullsized":"2024-09-13T14:23:59.639203+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644810+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:23:59.644810+0000","last_peered":"2024-09-13T14:23:59.644810+0000","last_clean":"2024-09-13T14:23:59.644810+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:23:59.644810+0000","last_undegraded":"2024-09-13T14:23:59.644810+0000","last_fullsized":"2024-09-13T14:23:59.644810+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643962+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:23:59.643962+0000","last_peered":"2024-09-13T14:23:59.643962+0000","last_clean":"2024-09-13T14:23:59.643962+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:23:59.643962+0000","last_undegraded":"2024-09-13T14:23:59.643962+0000","last_fullsized":"2024-09-13T14:23:59.643962+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645107+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:23:59.645107+0000","last_peered":"2024-09-13T14:23:59.645107+0000","last_clean":"2024-09-13T14:23:59.645107+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:23:59.645107+0000","last_undegraded":"2024-09-13T14:23:59.645107+0000","last_fullsized":"2024-09-13T14:23:59.645107+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652215+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:24:00.652215+0000","last_peered":"2024-09-13T14:24:00.652215+0000","last_clean":"2024-09-13T14:24:00.652215+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:24:00.652215+0000","last_undegraded":"2024-09-13T14:24:00.652215+0000","last_fullsized":"2024-09-13T14:24:00.652215+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":50,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645477+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:23:59.645477+0000","last_peered":"2024-09-13T14:23:59.645477+0000","last_clean":"2024-09-13T14:23:59.645477+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:23:59.645477+0000","last_undegraded":"2024-09-13T14:23:59.645477+0000","last_fullsized":"2024-09-13T14:23:59.645477+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640940+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:23:59.640940+0000","last_peered":"2024-09-13T14:23:59.640940+0000","last_clean":"2024-09-13T14:23:59.640940+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:23:59.640940+0000","last_undegraded":"2024-09-13T14:23:59.640940+0000","last_fullsized":"2024-09-13T14:23:59.640940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654168+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:24:00.654168+0000","last_peered":"2024-09-13T14:24:00.654168+0000","last_clean":"2024-09-13T14:24:00.654168+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:24:00.654168+0000","last_undegraded":"2024-09-13T14:24:00.654168+0000","last_fullsized":"2024-09-13T14:24:00.654168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643366+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:24:00.643366+0000","last_peered":"2024-09-13T14:24:00.643366+0000","last_clean":"2024-09-13T14:24:00.643366+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:24:00.643366+0000","last_undegraded":"2024-09-13T14:24:00.643366+0000","last_fullsized":"2024-09-13T14:24:00.643366+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.658797+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:24:00.658797+0000","last_peered":"2024-09-13T14:24:00.658797+0000","last_clean":"2024-09-13T14:24:00.658797+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:24:00.658797+0000","last_undegraded":"2024-09-13T14:24:00.658797+0000","last_fullsized":"2024-09-13T14:24:00.658797+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653246+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:24:00.653246+0000","last_peered":"2024-09-13T14:24:00.653246+0000","last_clean":"2024-09-13T14:24:00.653246+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:24:00.653246+0000","last_undegraded":"2024-09-13T14:24:00.653246+0000","last_fullsized":"2024-09-13T14:24:00.653246+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640237+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:23:59.640237+0000","last_peered":"2024-09-13T14:23:59.640237+0000","last_clean":"2024-09-13T14:23:59.640237+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:23:59.640237+0000","last_undegraded":"2024-09-13T14:23:59.640237+0000","last_fullsized":"2024-09-13T14:23:59.640237+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641057+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:23:59.641057+0000","last_peered":"2024-09-13T14:23:59.641057+0000","last_clean":"2024-09-13T14:23:59.641057+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:23:59.641057+0000","last_undegraded":"2024-09-13T14:23:59.641057+0000","last_fullsized":"2024-09-13T14:23:59.641057+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.642303+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:23:59.642303+0000","last_peered":"2024-09-13T14:23:59.642303+0000","last_clean":"2024-09-13T14:23:59.642303+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:23:59.642303+0000","last_undegraded":"2024-09-13T14:23:59.642303+0000","last_fullsized":"2024-09-13T14:23:59.642303+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640514+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:23:59.640514+0000","last_peered":"2024-09-13T14:23:59.640514+0000","last_clean":"2024-09-13T14:23:59.640514+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:23:59.640514+0000","last_undegraded":"2024-09-13T14:23:59.640514+0000","last_fullsized":"2024-09-13T14:23:59.640514+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652901+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:24:00.652901+0000","last_peered":"2024-09-13T14:24:00.652901+0000","last_clean":"2024-09-13T14:24:00.652901+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:24:00.652901+0000","last_undegraded":"2024-09-13T14:24:00.652901+0000","last_fullsized":"2024-09-13T14:24:00.652901+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654853+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:24:00.654853+0000","last_peered":"2024-09-13T14:24:00.654853+0000","last_clean":"2024-09-13T14:24:00.654853+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:24:00.654853+0000","last_undegraded":"2024-09-13T14:24:00.654853+0000","last_fullsized":"2024-09-13T14:24:00.654853+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":183,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644941+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:23:59.644941+0000","last_peered":"2024-09-13T14:23:59.644941+0000","last_clean":"2024-09-13T14:23:59.644941+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:23:59.644941+0000","last_undegraded":"2024-09-13T14:23:59.644941+0000","last_fullsized":"2024-09-13T14:23:59.644941+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":667,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644429+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:23:59.644429+0000","last_peered":"2024-09-13T14:23:59.644429+0000","last_clean":"2024-09-13T14:23:59.644429+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:23:59.644429+0000","last_undegraded":"2024-09-13T14:23:59.644429+0000","last_fullsized":"2024-09-13T14:23:59.644429+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653538+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:24:00.653538+0000","last_peered":"2024-09-13T14:24:00.653538+0000","last_clean":"2024-09-13T14:24:00.653538+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:24:00.653538+0000","last_undegraded":"2024-09-13T14:24:00.653538+0000","last_fullsized":"2024-09-13T14:24:00.653538+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641183+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:23:59.641183+0000","last_peered":"2024-09-13T14:23:59.641183+0000","last_clean":"2024-09-13T14:23:59.641183+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:23:59.641183+0000","last_undegraded":"2024-09-13T14:23:59.641183+0000","last_fullsized":"2024-09-13T14:23:59.641183+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":475,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644775+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:23:59.644775+0000","last_peered":"2024-09-13T14:23:59.644775+0000","last_clean":"2024-09-13T14:23:59.644775+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:23:59.644775+0000","last_undegraded":"2024-09-13T14:23:59.644775+0000","last_fullsized":"2024-09-13T14:23:59.644775+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653792+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:24:00.653792+0000","last_peered":"2024-09-13T14:24:00.653792+0000","last_clean":"2024-09-13T14:24:00.653792+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:24:00.653792+0000","last_undegraded":"2024-09-13T14:24:00.653792+0000","last_fullsized":"2024-09-13T14:24:00.653792+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.645745+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:24:00.645745+0000","last_peered":"2024-09-13T14:24:00.645745+0000","last_clean":"2024-09-13T14:24:00.645745+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:24:00.645745+0000","last_undegraded":"2024-09-13T14:24:00.645745+0000","last_fullsized":"2024-09-13T14:24:00.645745+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":494,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652899+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:24:00.652899+0000","last_peered":"2024-09-13T14:24:00.652899+0000","last_clean":"2024-09-13T14:24:00.652899+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:24:00.652899+0000","last_undegraded":"2024-09-13T14:24:00.652899+0000","last_fullsized":"2024-09-13T14:24:00.652899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643816+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:24:00.643816+0000","last_peered":"2024-09-13T14:24:00.643816+0000","last_clean":"2024-09-13T14:24:00.643816+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:24:00.643816+0000","last_undegraded":"2024-09-13T14:24:00.643816+0000","last_fullsized":"2024-09-13T14:24:00.643816+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.640811+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:24:00.640811+0000","last_peered":"2024-09-13T14:24:00.640811+0000","last_clean":"2024-09-13T14:24:00.640811+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:24:00.640811+0000","last_undegraded":"2024-09-13T14:24:00.640811+0000","last_fullsized":"2024-09-13T14:24:00.640811+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":716,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645356+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:23:59.645356+0000","last_peered":"2024-09-13T14:23:59.645356+0000","last_clean":"2024-09-13T14:23:59.645356+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:23:59.645356+0000","last_undegraded":"2024-09-13T14:23:59.645356+0000","last_fullsized":"2024-09-13T14:23:59.645356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641249+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:23:59.641249+0000","last_peered":"2024-09-13T14:23:59.641249+0000","last_clean":"2024-09-13T14:23:59.641249+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:23:59.641249+0000","last_undegraded":"2024-09-13T14:23:59.641249+0000","last_fullsized":"2024-09-13T14:23:59.641249+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655745+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:24:00.655745+0000","last_peered":"2024-09-13T14:24:00.655745+0000","last_clean":"2024-09-13T14:24:00.655745+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:24:00.655745+0000","last_undegraded":"2024-09-13T14:24:00.655745+0000","last_fullsized":"2024-09-13T14:24:00.655745+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652864+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:24:00.652864+0000","last_peered":"2024-09-13T14:24:00.652864+0000","last_clean":"2024-09-13T14:24:00.652864+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:24:00.652864+0000","last_undegraded":"2024-09-13T14:24:00.652864+0000","last_fullsized":"2024-09-13T14:24:00.652864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":665,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643732+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:24:00.643732+0000","last_peered":"2024-09-13T14:24:00.643732+0000","last_clean":"2024-09-13T14:24:00.643732+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:24:00.643732+0000","last_undegraded":"2024-09-13T14:24:00.643732+0000","last_fullsized":"2024-09-13T14:24:00.643732+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642248+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:24:00.642248+0000","last_peered":"2024-09-13T14:24:00.642248+0000","last_clean":"2024-09-13T14:24:00.642248+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:24:00.642248+0000","last_undegraded":"2024-09-13T14:24:00.642248+0000","last_fullsized":"2024-09-13T14:24:00.642248+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643629+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:24:00.643629+0000","last_peered":"2024-09-13T14:24:00.643629+0000","last_clean":"2024-09-13T14:24:00.643629+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:24:00.643629+0000","last_undegraded":"2024-09-13T14:24:00.643629+0000","last_fullsized":"2024-09-13T14:24:00.643629+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":412,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645552+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:23:59.645552+0000","last_peered":"2024-09-13T14:23:59.645552+0000","last_clean":"2024-09-13T14:23:59.645552+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:23:59.645552+0000","last_undegraded":"2024-09-13T14:23:59.645552+0000","last_fullsized":"2024-09-13T14:23:59.645552+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643279+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:24:00.643279+0000","last_peered":"2024-09-13T14:24:00.643279+0000","last_clean":"2024-09-13T14:24:00.643279+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:24:00.643279+0000","last_undegraded":"2024-09-13T14:24:00.643279+0000","last_fullsized":"2024-09-13T14:24:00.643279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659609+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:24:00.659609+0000","last_peered":"2024-09-13T14:24:00.659609+0000","last_clean":"2024-09-13T14:24:00.659609+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:24:00.659609+0000","last_undegraded":"2024-09-13T14:24:00.659609+0000","last_fullsized":"2024-09-13T14:24:00.659609+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":398,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642581+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:24:00.642581+0000","last_peered":"2024-09-13T14:24:00.642581+0000","last_clean":"2024-09-13T14:24:00.642581+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:24:00.642581+0000","last_undegraded":"2024-09-13T14:24:00.642581+0000","last_fullsized":"2024-09-13T14:24:00.642581+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652559+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:24:00.652559+0000","last_peered":"2024-09-13T14:24:00.652559+0000","last_clean":"2024-09-13T14:24:00.652559+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:24:00.652559+0000","last_undegraded":"2024-09-13T14:24:00.652559+0000","last_fullsized":"2024-09-13T14:24:00.652559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643647+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:24:00.643647+0000","last_peered":"2024-09-13T14:24:00.643647+0000","last_clean":"2024-09-13T14:24:00.643647+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:24:00.643647+0000","last_undegraded":"2024-09-13T14:24:00.643647+0000","last_fullsized":"2024-09-13T14:24:00.643647+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":520,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641454+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:23:59.641454+0000","last_peered":"2024-09-13T14:23:59.641454+0000","last_clean":"2024-09-13T14:23:59.641454+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:23:59.641454+0000","last_undegraded":"2024-09-13T14:23:59.641454+0000","last_fullsized":"2024-09-13T14:23:59.641454+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639117+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:23:59.639117+0000","last_peered":"2024-09-13T14:23:59.639117+0000","last_clean":"2024-09-13T14:23:59.639117+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:23:59.639117+0000","last_undegraded":"2024-09-13T14:23:59.639117+0000","last_fullsized":"2024-09-13T14:23:59.639117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641076+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:23:59.641076+0000","last_peered":"2024-09-13T14:23:59.641076+0000","last_clean":"2024-09-13T14:23:59.641076+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:23:59.641076+0000","last_undegraded":"2024-09-13T14:23:59.641076+0000","last_fullsized":"2024-09-13T14:23:59.641076+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":72,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639435+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:23:59.639435+0000","last_peered":"2024-09-13T14:23:59.639435+0000","last_clean":"2024-09-13T14:23:59.639435+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:23:59.639435+0000","last_undegraded":"2024-09-13T14:23:59.639435+0000","last_fullsized":"2024-09-13T14:23:59.639435+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642056+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:24:00.642056+0000","last_peered":"2024-09-13T14:24:00.642056+0000","last_clean":"2024-09-13T14:24:00.642056+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:24:00.642056+0000","last_undegraded":"2024-09-13T14:24:00.642056+0000","last_fullsized":"2024-09-13T14:24:00.642056+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.650848+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:24:00.650848+0000","last_peered":"2024-09-13T14:24:00.650848+0000","last_clean":"2024-09-13T14:24:00.650848+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:24:00.650848+0000","last_undegraded":"2024-09-13T14:24:00.650848+0000","last_fullsized":"2024-09-13T14:24:00.650848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644269+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:23:59.644269+0000","last_peered":"2024-09-13T14:23:59.644269+0000","last_clean":"2024-09-13T14:23:59.644269+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:23:59.644269+0000","last_undegraded":"2024-09-13T14:23:59.644269+0000","last_fullsized":"2024-09-13T14:23:59.644269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657819+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:24:00.657819+0000","last_peered":"2024-09-13T14:24:00.657819+0000","last_clean":"2024-09-13T14:24:00.657819+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:24:00.657819+0000","last_undegraded":"2024-09-13T14:24:00.657819+0000","last_fullsized":"2024-09-13T14:24:00.657819+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655260+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:24:00.655260+0000","last_peered":"2024-09-13T14:24:00.655260+0000","last_clean":"2024-09-13T14:24:00.655260+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:24:00.655260+0000","last_undegraded":"2024-09-13T14:24:00.655260+0000","last_fullsized":"2024-09-13T14:24:00.655260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653009+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:24:00.653009+0000","last_peered":"2024-09-13T14:24:00.653009+0000","last_clean":"2024-09-13T14:24:00.653009+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:24:00.653009+0000","last_undegraded":"2024-09-13T14:24:00.653009+0000","last_fullsized":"2024-09-13T14:24:00.653009+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641367+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:23:59.641367+0000","last_peered":"2024-09-13T14:23:59.641367+0000","last_clean":"2024-09-13T14:23:59.641367+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:23:59.641367+0000","last_undegraded":"2024-09-13T14:23:59.641367+0000","last_fullsized":"2024-09-13T14:23:59.641367+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652277+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:24:00.652277+0000","last_peered":"2024-09-13T14:24:00.652277+0000","last_clean":"2024-09-13T14:24:00.652277+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:24:00.652277+0000","last_undegraded":"2024-09-13T14:24:00.652277+0000","last_fullsized":"2024-09-13T14:24:00.652277+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657178+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:24:00.657178+0000","last_peered":"2024-09-13T14:24:00.657178+0000","last_clean":"2024-09-13T14:24:00.657178+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:24:00.657178+0000","last_undegraded":"2024-09-13T14:24:00.657178+0000","last_fullsized":"2024-09-13T14:24:00.657178+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654244+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:24:00.654244+0000","last_peered":"2024-09-13T14:24:00.654244+0000","last_clean":"2024-09-13T14:24:00.654244+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:24:00.654244+0000","last_undegraded":"2024-09-13T14:24:00.654244+0000","last_fullsized":"2024-09-13T14:24:00.654244+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653972+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:24:00.653972+0000","last_peered":"2024-09-13T14:24:00.653972+0000","last_clean":"2024-09-13T14:24:00.653972+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:24:00.653972+0000","last_undegraded":"2024-09-13T14:24:00.653972+0000","last_fullsized":"2024-09-13T14:24:00.653972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":85,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639960+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:23:59.639960+0000","last_peered":"2024-09-13T14:23:59.639960+0000","last_clean":"2024-09-13T14:23:59.639960+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:23:59.639960+0000","last_undegraded":"2024-09-13T14:23:59.639960+0000","last_fullsized":"2024-09-13T14:23:59.639960+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654744+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:24:00.654744+0000","last_peered":"2024-09-13T14:24:00.654744+0000","last_clean":"2024-09-13T14:24:00.654744+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:24:00.654744+0000","last_undegraded":"2024-09-13T14:24:00.654744+0000","last_fullsized":"2024-09-13T14:24:00.654744+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":58,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643227+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:24:00.643227+0000","last_peered":"2024-09-13T14:24:00.643227+0000","last_clean":"2024-09-13T14:24:00.643227+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:24:00.643227+0000","last_undegraded":"2024-09-13T14:24:00.643227+0000","last_fullsized":"2024-09-13T14:24:00.643227+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652248+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:24:00.652248+0000","last_peered":"2024-09-13T14:24:00.652248+0000","last_clean":"2024-09-13T14:24:00.652248+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:24:00.652248+0000","last_undegraded":"2024-09-13T14:24:00.652248+0000","last_fullsized":"2024-09-13T14:24:00.652248+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.646313+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:23:59.646313+0000","last_peered":"2024-09-13T14:23:59.646313+0000","last_clean":"2024-09-13T14:23:59.646313+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:23:59.646313+0000","last_undegraded":"2024-09-13T14:23:59.646313+0000","last_fullsized":"2024-09-13T14:23:59.646313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652085+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:24:00.652085+0000","last_peered":"2024-09-13T14:24:00.652085+0000","last_clean":"2024-09-13T14:24:00.652085+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:24:00.652085+0000","last_undegraded":"2024-09-13T14:24:00.652085+0000","last_fullsized":"2024-09-13T14:24:00.652085+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":51,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645618+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:23:59.645618+0000","last_peered":"2024-09-13T14:23:59.645618+0000","last_clean":"2024-09-13T14:23:59.645618+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:23:59.645618+0000","last_undegraded":"2024-09-13T14:23:59.645618+0000","last_fullsized":"2024-09-13T14:23:59.645618+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659564+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:24:00.659564+0000","last_peered":"2024-09-13T14:24:00.659564+0000","last_clean":"2024-09-13T14:24:00.659564+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:24:00.659564+0000","last_undegraded":"2024-09-13T14:24:00.659564+0000","last_fullsized":"2024-09-13T14:24:00.659564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":75,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654100+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:24:00.654100+0000","last_peered":"2024-09-13T14:24:00.654100+0000","last_clean":"2024-09-13T14:24:00.654100+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:24:00.654100+0000","last_undegraded":"2024-09-13T14:24:00.654100+0000","last_fullsized":"2024-09-13T14:24:00.654100+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639012+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:23:59.639012+0000","last_peered":"2024-09-13T14:23:59.639012+0000","last_clean":"2024-09-13T14:23:59.639012+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:23:59.639012+0000","last_undegraded":"2024-09-13T14:23:59.639012+0000","last_fullsized":"2024-09-13T14:23:59.639012+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.644014+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:24:00.644014+0000","last_peered":"2024-09-13T14:24:00.644014+0000","last_clean":"2024-09-13T14:24:00.644014+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:24:00.644014+0000","last_undegraded":"2024-09-13T14:24:00.644014+0000","last_fullsized":"2024-09-13T14:24:00.644014+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397666,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307316,"kb_used_data":5792,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437836,"statfs":{"total":95995035648,"available":95680344064,"internally_reserved":0,"allocated":5931008,"data_stored":5374887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593894,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299044,"kb_used_data":1680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446108,"statfs":{"total":95995035648,"available":95688814592,"internally_reserved":0,"allocated":1720320,"data_stored":1179889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790124,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298456,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446696,"statfs":{"total":95995035648,"available":95689416704,"internally_reserved":0,"allocated":1118208,"data_stored":585284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986352,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298468,"kb_used_data":1112,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446684,"statfs":{"total":95995035648,"available":95689404416,"internally_reserved":0,"allocated":1138688,"data_stored":590145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696909,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1092,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1118208,"data_stored":585357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":28109899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860436,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1096,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1122304,"data_stored":585879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":32431948},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842023963,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36880,"kb_used_data":5188,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708272,"statfs":{"total":95995035648,"available":95957270528,"internally_reserved":0,"allocated":5312512,"data_stored":4779582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":32432567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187498,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":5768,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":5906432,"data_stored":5370224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32432459},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:24:59.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:24:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:24:59 smithi189 ceph-mon[90992]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:24:59.765 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:00.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:00 smithi189 ceph-mon[90992]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:00.568 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:00 smithi017 ceph-mon[103001]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:00.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:00 smithi017 ceph-mon[107556]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:01.492 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:25:01.493 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:25:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:01 smithi017 ceph-mon[103001]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:01 smithi017 ceph-mon[107556]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:01 smithi189 ceph-mon[90992]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:02.162 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":227,"stamp":"2024-09-13T14:24:59.723587+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338996,"kb_used_data":22820,"kb_used_omap":35,"kb_used_meta":1315996,"kb_avail":748622220,"statfs":{"total":767960285184,"available":766589153280,"internally_reserved":0,"allocated":23367680,"data_stored":19051247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1347580921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003000"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.661336+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:24:00.661336+0000","last_peered":"2024-09-13T14:24:00.661336+0000","last_clean":"2024-09-13T14:24:00.661336+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:24:00.661336+0000","last_undegraded":"2024-09-13T14:24:00.661336+0000","last_fullsized":"2024-09-13T14:24:00.661336+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644047+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:23:59.644047+0000","last_peered":"2024-09-13T14:23:59.644047+0000","last_clean":"2024-09-13T14:23:59.644047+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:23:59.644047+0000","last_undegraded":"2024-09-13T14:23:59.644047+0000","last_fullsized":"2024-09-13T14:23:59.644047+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653974+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:24:00.653974+0000","last_peered":"2024-09-13T14:24:00.653974+0000","last_clean":"2024-09-13T14:24:00.653974+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:24:00.653974+0000","last_undegraded":"2024-09-13T14:24:00.653974+0000","last_fullsized":"2024-09-13T14:24:00.653974+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655023+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:24:00.655023+0000","last_peered":"2024-09-13T14:24:00.655023+0000","last_clean":"2024-09-13T14:24:00.655023+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:24:00.655023+0000","last_undegraded":"2024-09-13T14:24:00.655023+0000","last_fullsized":"2024-09-13T14:24:00.655023+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657045+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:24:00.657045+0000","last_peered":"2024-09-13T14:24:00.657045+0000","last_clean":"2024-09-13T14:24:00.657045+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:24:00.657045+0000","last_undegraded":"2024-09-13T14:24:00.657045+0000","last_fullsized":"2024-09-13T14:24:00.657045+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643390+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:23:59.643390+0000","last_peered":"2024-09-13T14:23:59.643390+0000","last_clean":"2024-09-13T14:23:59.643390+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:23:59.643390+0000","last_undegraded":"2024-09-13T14:23:59.643390+0000","last_fullsized":"2024-09-13T14:23:59.643390+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.651402+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:24:00.651402+0000","last_peered":"2024-09-13T14:24:00.651402+0000","last_clean":"2024-09-13T14:24:00.651402+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:24:00.651402+0000","last_undegraded":"2024-09-13T14:24:00.651402+0000","last_fullsized":"2024-09-13T14:24:00.651402+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":74,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639203+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:23:59.639203+0000","last_peered":"2024-09-13T14:23:59.639203+0000","last_clean":"2024-09-13T14:23:59.639203+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:23:59.639203+0000","last_undegraded":"2024-09-13T14:23:59.639203+0000","last_fullsized":"2024-09-13T14:23:59.639203+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644810+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:23:59.644810+0000","last_peered":"2024-09-13T14:23:59.644810+0000","last_clean":"2024-09-13T14:23:59.644810+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:23:59.644810+0000","last_undegraded":"2024-09-13T14:23:59.644810+0000","last_fullsized":"2024-09-13T14:23:59.644810+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643962+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:23:59.643962+0000","last_peered":"2024-09-13T14:23:59.643962+0000","last_clean":"2024-09-13T14:23:59.643962+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:23:59.643962+0000","last_undegraded":"2024-09-13T14:23:59.643962+0000","last_fullsized":"2024-09-13T14:23:59.643962+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645107+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:23:59.645107+0000","last_peered":"2024-09-13T14:23:59.645107+0000","last_clean":"2024-09-13T14:23:59.645107+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:23:59.645107+0000","last_undegraded":"2024-09-13T14:23:59.645107+0000","last_fullsized":"2024-09-13T14:23:59.645107+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652215+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:24:00.652215+0000","last_peered":"2024-09-13T14:24:00.652215+0000","last_clean":"2024-09-13T14:24:00.652215+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:24:00.652215+0000","last_undegraded":"2024-09-13T14:24:00.652215+0000","last_fullsized":"2024-09-13T14:24:00.652215+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":50,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645477+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:23:59.645477+0000","last_peered":"2024-09-13T14:23:59.645477+0000","last_clean":"2024-09-13T14:23:59.645477+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:23:59.645477+0000","last_undegraded":"2024-09-13T14:23:59.645477+0000","last_fullsized":"2024-09-13T14:23:59.645477+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640940+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:23:59.640940+0000","last_peered":"2024-09-13T14:23:59.640940+0000","last_clean":"2024-09-13T14:23:59.640940+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:23:59.640940+0000","last_undegraded":"2024-09-13T14:23:59.640940+0000","last_fullsized":"2024-09-13T14:23:59.640940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654168+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:24:00.654168+0000","last_peered":"2024-09-13T14:24:00.654168+0000","last_clean":"2024-09-13T14:24:00.654168+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:24:00.654168+0000","last_undegraded":"2024-09-13T14:24:00.654168+0000","last_fullsized":"2024-09-13T14:24:00.654168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643366+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:24:00.643366+0000","last_peered":"2024-09-13T14:24:00.643366+0000","last_clean":"2024-09-13T14:24:00.643366+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:24:00.643366+0000","last_undegraded":"2024-09-13T14:24:00.643366+0000","last_fullsized":"2024-09-13T14:24:00.643366+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.658797+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:24:00.658797+0000","last_peered":"2024-09-13T14:24:00.658797+0000","last_clean":"2024-09-13T14:24:00.658797+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:24:00.658797+0000","last_undegraded":"2024-09-13T14:24:00.658797+0000","last_fullsized":"2024-09-13T14:24:00.658797+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653246+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:24:00.653246+0000","last_peered":"2024-09-13T14:24:00.653246+0000","last_clean":"2024-09-13T14:24:00.653246+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:24:00.653246+0000","last_undegraded":"2024-09-13T14:24:00.653246+0000","last_fullsized":"2024-09-13T14:24:00.653246+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640237+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:23:59.640237+0000","last_peered":"2024-09-13T14:23:59.640237+0000","last_clean":"2024-09-13T14:23:59.640237+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:23:59.640237+0000","last_undegraded":"2024-09-13T14:23:59.640237+0000","last_fullsized":"2024-09-13T14:23:59.640237+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641057+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:23:59.641057+0000","last_peered":"2024-09-13T14:23:59.641057+0000","last_clean":"2024-09-13T14:23:59.641057+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:23:59.641057+0000","last_undegraded":"2024-09-13T14:23:59.641057+0000","last_fullsized":"2024-09-13T14:23:59.641057+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.642303+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:23:59.642303+0000","last_peered":"2024-09-13T14:23:59.642303+0000","last_clean":"2024-09-13T14:23:59.642303+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:23:59.642303+0000","last_undegraded":"2024-09-13T14:23:59.642303+0000","last_fullsized":"2024-09-13T14:23:59.642303+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640514+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:23:59.640514+0000","last_peered":"2024-09-13T14:23:59.640514+0000","last_clean":"2024-09-13T14:23:59.640514+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:23:59.640514+0000","last_undegraded":"2024-09-13T14:23:59.640514+0000","last_fullsized":"2024-09-13T14:23:59.640514+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652901+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:24:00.652901+0000","last_peered":"2024-09-13T14:24:00.652901+0000","last_clean":"2024-09-13T14:24:00.652901+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:24:00.652901+0000","last_undegraded":"2024-09-13T14:24:00.652901+0000","last_fullsized":"2024-09-13T14:24:00.652901+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654853+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:24:00.654853+0000","last_peered":"2024-09-13T14:24:00.654853+0000","last_clean":"2024-09-13T14:24:00.654853+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:24:00.654853+0000","last_undegraded":"2024-09-13T14:24:00.654853+0000","last_fullsized":"2024-09-13T14:24:00.654853+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":183,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644941+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:23:59.644941+0000","last_peered":"2024-09-13T14:23:59.644941+0000","last_clean":"2024-09-13T14:23:59.644941+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:23:59.644941+0000","last_undegraded":"2024-09-13T14:23:59.644941+0000","last_fullsized":"2024-09-13T14:23:59.644941+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":667,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644429+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:23:59.644429+0000","last_peered":"2024-09-13T14:23:59.644429+0000","last_clean":"2024-09-13T14:23:59.644429+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:23:59.644429+0000","last_undegraded":"2024-09-13T14:23:59.644429+0000","last_fullsized":"2024-09-13T14:23:59.644429+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653538+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:24:00.653538+0000","last_peered":"2024-09-13T14:24:00.653538+0000","last_clean":"2024-09-13T14:24:00.653538+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:24:00.653538+0000","last_undegraded":"2024-09-13T14:24:00.653538+0000","last_fullsized":"2024-09-13T14:24:00.653538+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641183+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:23:59.641183+0000","last_peered":"2024-09-13T14:23:59.641183+0000","last_clean":"2024-09-13T14:23:59.641183+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:23:59.641183+0000","last_undegraded":"2024-09-13T14:23:59.641183+0000","last_fullsized":"2024-09-13T14:23:59.641183+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":475,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644775+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:23:59.644775+0000","last_peered":"2024-09-13T14:23:59.644775+0000","last_clean":"2024-09-13T14:23:59.644775+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:23:59.644775+0000","last_undegraded":"2024-09-13T14:23:59.644775+0000","last_fullsized":"2024-09-13T14:23:59.644775+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653792+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:24:00.653792+0000","last_peered":"2024-09-13T14:24:00.653792+0000","last_clean":"2024-09-13T14:24:00.653792+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:24:00.653792+0000","last_undegraded":"2024-09-13T14:24:00.653792+0000","last_fullsized":"2024-09-13T14:24:00.653792+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.645745+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:24:00.645745+0000","last_peered":"2024-09-13T14:24:00.645745+0000","last_clean":"2024-09-13T14:24:00.645745+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:24:00.645745+0000","last_undegraded":"2024-09-13T14:24:00.645745+0000","last_fullsized":"2024-09-13T14:24:00.645745+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":494,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652899+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:24:00.652899+0000","last_peered":"2024-09-13T14:24:00.652899+0000","last_clean":"2024-09-13T14:24:00.652899+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:24:00.652899+0000","last_undegraded":"2024-09-13T14:24:00.652899+0000","last_fullsized":"2024-09-13T14:24:00.652899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643816+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:24:00.643816+0000","last_peered":"2024-09-13T14:24:00.643816+0000","last_clean":"2024-09-13T14:24:00.643816+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:24:00.643816+0000","last_undegraded":"2024-09-13T14:24:00.643816+0000","last_fullsized":"2024-09-13T14:24:00.643816+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.640811+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:24:00.640811+0000","last_peered":"2024-09-13T14:24:00.640811+0000","last_clean":"2024-09-13T14:24:00.640811+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:24:00.640811+0000","last_undegraded":"2024-09-13T14:24:00.640811+0000","last_fullsized":"2024-09-13T14:24:00.640811+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":716,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645356+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:23:59.645356+0000","last_peered":"2024-09-13T14:23:59.645356+0000","last_clean":"2024-09-13T14:23:59.645356+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:23:59.645356+0000","last_undegraded":"2024-09-13T14:23:59.645356+0000","last_fullsized":"2024-09-13T14:23:59.645356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641249+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:23:59.641249+0000","last_peered":"2024-09-13T14:23:59.641249+0000","last_clean":"2024-09-13T14:23:59.641249+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:23:59.641249+0000","last_undegraded":"2024-09-13T14:23:59.641249+0000","last_fullsized":"2024-09-13T14:23:59.641249+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655745+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:24:00.655745+0000","last_peered":"2024-09-13T14:24:00.655745+0000","last_clean":"2024-09-13T14:24:00.655745+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:24:00.655745+0000","last_undegraded":"2024-09-13T14:24:00.655745+0000","last_fullsized":"2024-09-13T14:24:00.655745+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652864+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:24:00.652864+0000","last_peered":"2024-09-13T14:24:00.652864+0000","last_clean":"2024-09-13T14:24:00.652864+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:24:00.652864+0000","last_undegraded":"2024-09-13T14:24:00.652864+0000","last_fullsized":"2024-09-13T14:24:00.652864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":665,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643732+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:24:00.643732+0000","last_peered":"2024-09-13T14:24:00.643732+0000","last_clean":"2024-09-13T14:24:00.643732+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:24:00.643732+0000","last_undegraded":"2024-09-13T14:24:00.643732+0000","last_fullsized":"2024-09-13T14:24:00.643732+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642248+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:24:00.642248+0000","last_peered":"2024-09-13T14:24:00.642248+0000","last_clean":"2024-09-13T14:24:00.642248+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:24:00.642248+0000","last_undegraded":"2024-09-13T14:24:00.642248+0000","last_fullsized":"2024-09-13T14:24:00.642248+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643629+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:24:00.643629+0000","last_peered":"2024-09-13T14:24:00.643629+0000","last_clean":"2024-09-13T14:24:00.643629+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:24:00.643629+0000","last_undegraded":"2024-09-13T14:24:00.643629+0000","last_fullsized":"2024-09-13T14:24:00.643629+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":412,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645552+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:23:59.645552+0000","last_peered":"2024-09-13T14:23:59.645552+0000","last_clean":"2024-09-13T14:23:59.645552+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:23:59.645552+0000","last_undegraded":"2024-09-13T14:23:59.645552+0000","last_fullsized":"2024-09-13T14:23:59.645552+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643279+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:24:00.643279+0000","last_peered":"2024-09-13T14:24:00.643279+0000","last_clean":"2024-09-13T14:24:00.643279+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:24:00.643279+0000","last_undegraded":"2024-09-13T14:24:00.643279+0000","last_fullsized":"2024-09-13T14:24:00.643279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659609+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:24:00.659609+0000","last_peered":"2024-09-13T14:24:00.659609+0000","last_clean":"2024-09-13T14:24:00.659609+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:24:00.659609+0000","last_undegraded":"2024-09-13T14:24:00.659609+0000","last_fullsized":"2024-09-13T14:24:00.659609+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":398,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642581+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:24:00.642581+0000","last_peered":"2024-09-13T14:24:00.642581+0000","last_clean":"2024-09-13T14:24:00.642581+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:24:00.642581+0000","last_undegraded":"2024-09-13T14:24:00.642581+0000","last_fullsized":"2024-09-13T14:24:00.642581+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652559+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:24:00.652559+0000","last_peered":"2024-09-13T14:24:00.652559+0000","last_clean":"2024-09-13T14:24:00.652559+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:24:00.652559+0000","last_undegraded":"2024-09-13T14:24:00.652559+0000","last_fullsized":"2024-09-13T14:24:00.652559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643647+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:24:00.643647+0000","last_peered":"2024-09-13T14:24:00.643647+0000","last_clean":"2024-09-13T14:24:00.643647+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:24:00.643647+0000","last_undegraded":"2024-09-13T14:24:00.643647+0000","last_fullsized":"2024-09-13T14:24:00.643647+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":520,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641454+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:23:59.641454+0000","last_peered":"2024-09-13T14:23:59.641454+0000","last_clean":"2024-09-13T14:23:59.641454+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:23:59.641454+0000","last_undegraded":"2024-09-13T14:23:59.641454+0000","last_fullsized":"2024-09-13T14:23:59.641454+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639117+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:23:59.639117+0000","last_peered":"2024-09-13T14:23:59.639117+0000","last_clean":"2024-09-13T14:23:59.639117+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:23:59.639117+0000","last_undegraded":"2024-09-13T14:23:59.639117+0000","last_fullsized":"2024-09-13T14:23:59.639117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641076+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:23:59.641076+0000","last_peered":"2024-09-13T14:23:59.641076+0000","last_clean":"2024-09-13T14:23:59.641076+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:23:59.641076+0000","last_undegraded":"2024-09-13T14:23:59.641076+0000","last_fullsized":"2024-09-13T14:23:59.641076+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":72,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639435+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:23:59.639435+0000","last_peered":"2024-09-13T14:23:59.639435+0000","last_clean":"2024-09-13T14:23:59.639435+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:23:59.639435+0000","last_undegraded":"2024-09-13T14:23:59.639435+0000","last_fullsized":"2024-09-13T14:23:59.639435+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642056+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:24:00.642056+0000","last_peered":"2024-09-13T14:24:00.642056+0000","last_clean":"2024-09-13T14:24:00.642056+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:24:00.642056+0000","last_undegraded":"2024-09-13T14:24:00.642056+0000","last_fullsized":"2024-09-13T14:24:00.642056+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.650848+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:24:00.650848+0000","last_peered":"2024-09-13T14:24:00.650848+0000","last_clean":"2024-09-13T14:24:00.650848+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:24:00.650848+0000","last_undegraded":"2024-09-13T14:24:00.650848+0000","last_fullsized":"2024-09-13T14:24:00.650848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644269+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:23:59.644269+0000","last_peered":"2024-09-13T14:23:59.644269+0000","last_clean":"2024-09-13T14:23:59.644269+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:23:59.644269+0000","last_undegraded":"2024-09-13T14:23:59.644269+0000","last_fullsized":"2024-09-13T14:23:59.644269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657819+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:24:00.657819+0000","last_peered":"2024-09-13T14:24:00.657819+0000","last_clean":"2024-09-13T14:24:00.657819+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:24:00.657819+0000","last_undegraded":"2024-09-13T14:24:00.657819+0000","last_fullsized":"2024-09-13T14:24:00.657819+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655260+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:24:00.655260+0000","last_peered":"2024-09-13T14:24:00.655260+0000","last_clean":"2024-09-13T14:24:00.655260+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:24:00.655260+0000","last_undegraded":"2024-09-13T14:24:00.655260+0000","last_fullsized":"2024-09-13T14:24:00.655260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653009+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:24:00.653009+0000","last_peered":"2024-09-13T14:24:00.653009+0000","last_clean":"2024-09-13T14:24:00.653009+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:24:00.653009+0000","last_undegraded":"2024-09-13T14:24:00.653009+0000","last_fullsized":"2024-09-13T14:24:00.653009+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641367+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:23:59.641367+0000","last_peered":"2024-09-13T14:23:59.641367+0000","last_clean":"2024-09-13T14:23:59.641367+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:23:59.641367+0000","last_undegraded":"2024-09-13T14:23:59.641367+0000","last_fullsized":"2024-09-13T14:23:59.641367+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652277+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:24:00.652277+0000","last_peered":"2024-09-13T14:24:00.652277+0000","last_clean":"2024-09-13T14:24:00.652277+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:24:00.652277+0000","last_undegraded":"2024-09-13T14:24:00.652277+0000","last_fullsized":"2024-09-13T14:24:00.652277+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657178+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:24:00.657178+0000","last_peered":"2024-09-13T14:24:00.657178+0000","last_clean":"2024-09-13T14:24:00.657178+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:24:00.657178+0000","last_undegraded":"2024-09-13T14:24:00.657178+0000","last_fullsized":"2024-09-13T14:24:00.657178+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654244+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:24:00.654244+0000","last_peered":"2024-09-13T14:24:00.654244+0000","last_clean":"2024-09-13T14:24:00.654244+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:24:00.654244+0000","last_undegraded":"2024-09-13T14:24:00.654244+0000","last_fullsized":"2024-09-13T14:24:00.654244+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653972+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:24:00.653972+0000","last_peered":"2024-09-13T14:24:00.653972+0000","last_clean":"2024-09-13T14:24:00.653972+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:24:00.653972+0000","last_undegraded":"2024-09-13T14:24:00.653972+0000","last_fullsized":"2024-09-13T14:24:00.653972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":85,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639960+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:23:59.639960+0000","last_peered":"2024-09-13T14:23:59.639960+0000","last_clean":"2024-09-13T14:23:59.639960+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:23:59.639960+0000","last_undegraded":"2024-09-13T14:23:59.639960+0000","last_fullsized":"2024-09-13T14:23:59.639960+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654744+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:24:00.654744+0000","last_peered":"2024-09-13T14:24:00.654744+0000","last_clean":"2024-09-13T14:24:00.654744+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:24:00.654744+0000","last_undegraded":"2024-09-13T14:24:00.654744+0000","last_fullsized":"2024-09-13T14:24:00.654744+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":58,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643227+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:24:00.643227+0000","last_peered":"2024-09-13T14:24:00.643227+0000","last_clean":"2024-09-13T14:24:00.643227+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:24:00.643227+0000","last_undegraded":"2024-09-13T14:24:00.643227+0000","last_fullsized":"2024-09-13T14:24:00.643227+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652248+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:24:00.652248+0000","last_peered":"2024-09-13T14:24:00.652248+0000","last_clean":"2024-09-13T14:24:00.652248+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:24:00.652248+0000","last_undegraded":"2024-09-13T14:24:00.652248+0000","last_fullsized":"2024-09-13T14:24:00.652248+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.646313+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:23:59.646313+0000","last_peered":"2024-09-13T14:23:59.646313+0000","last_clean":"2024-09-13T14:23:59.646313+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:23:59.646313+0000","last_undegraded":"2024-09-13T14:23:59.646313+0000","last_fullsized":"2024-09-13T14:23:59.646313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652085+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:24:00.652085+0000","last_peered":"2024-09-13T14:24:00.652085+0000","last_clean":"2024-09-13T14:24:00.652085+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:24:00.652085+0000","last_undegraded":"2024-09-13T14:24:00.652085+0000","last_fullsized":"2024-09-13T14:24:00.652085+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":51,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645618+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:23:59.645618+0000","last_peered":"2024-09-13T14:23:59.645618+0000","last_clean":"2024-09-13T14:23:59.645618+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:23:59.645618+0000","last_undegraded":"2024-09-13T14:23:59.645618+0000","last_fullsized":"2024-09-13T14:23:59.645618+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659564+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:24:00.659564+0000","last_peered":"2024-09-13T14:24:00.659564+0000","last_clean":"2024-09-13T14:24:00.659564+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:24:00.659564+0000","last_undegraded":"2024-09-13T14:24:00.659564+0000","last_fullsized":"2024-09-13T14:24:00.659564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":75,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654100+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:24:00.654100+0000","last_peered":"2024-09-13T14:24:00.654100+0000","last_clean":"2024-09-13T14:24:00.654100+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:24:00.654100+0000","last_undegraded":"2024-09-13T14:24:00.654100+0000","last_fullsized":"2024-09-13T14:24:00.654100+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639012+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:23:59.639012+0000","last_peered":"2024-09-13T14:23:59.639012+0000","last_clean":"2024-09-13T14:23:59.639012+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:23:59.639012+0000","last_undegraded":"2024-09-13T14:23:59.639012+0000","last_fullsized":"2024-09-13T14:23:59.639012+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.644014+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:24:00.644014+0000","last_peered":"2024-09-13T14:24:00.644014+0000","last_clean":"2024-09-13T14:24:00.644014+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:24:00.644014+0000","last_undegraded":"2024-09-13T14:24:00.644014+0000","last_fullsized":"2024-09-13T14:24:00.644014+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397666,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307316,"kb_used_data":5792,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437836,"statfs":{"total":95995035648,"available":95680344064,"internally_reserved":0,"allocated":5931008,"data_stored":5374887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593894,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299044,"kb_used_data":1680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446108,"statfs":{"total":95995035648,"available":95688814592,"internally_reserved":0,"allocated":1720320,"data_stored":1179889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790124,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298456,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446696,"statfs":{"total":95995035648,"available":95689416704,"internally_reserved":0,"allocated":1118208,"data_stored":585284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986352,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298468,"kb_used_data":1112,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446684,"statfs":{"total":95995035648,"available":95689404416,"internally_reserved":0,"allocated":1138688,"data_stored":590145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696910,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1092,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1118208,"data_stored":585357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":28109899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860436,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1096,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1122304,"data_stored":585879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":32431948},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842023963,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36880,"kb_used_data":5188,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708272,"statfs":{"total":95995035648,"available":95957270528,"internally_reserved":0,"allocated":5312512,"data_stored":4779582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":32432567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187498,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":5768,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":5906432,"data_stored":5370224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32432459},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:25:02.165 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:25:02.165 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:25:02.165 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:25:02.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:25:02.430 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:03 smithi017 ceph-mon[103001]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:03 smithi017 ceph-mon[103001]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:03 smithi017 ceph-mon[107556]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:03 smithi017 ceph-mon[107556]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:03 smithi189 ceph-mon[90992]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:03 smithi189 ceph-mon[90992]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:04.276 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:25:04.276 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:25:04.639 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2409237481' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:25:04.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2409237481' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:25:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2409237481' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:25:05.067 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:25:05.067 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:25:05.077 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-13T14:25:05.077 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:25:05.090 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:25:05.090 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:25:05.091 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-13T14:25:05.093 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:25:05.093 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-aclamk-cbt-combined'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-aclamk-cbt-combined', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:25:05.093 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:25:05.093 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-13T14:25:05.093 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:25:05.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:25:05.351 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:05.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:05 smithi017 ceph-mon[103001]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:05 smithi017 ceph-mon[107556]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:05 smithi189 ceph-mon[90992]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:07.115 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:25:07.116 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:25:07.397 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:07] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.51.0" 2024-09-13T14:25:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:07 smithi017 ceph-mon[103001]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:07 smithi017 ceph-mon[107556]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:07 smithi189 ceph-mon[90992]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:07.782 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":230,"stamp":"2024-09-13T14:25:05.724855+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338996,"kb_used_data":22820,"kb_used_omap":35,"kb_used_meta":1315996,"kb_avail":748622220,"statfs":{"total":767960285184,"available":766589153280,"internally_reserved":0,"allocated":23367680,"data_stored":19051247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1347580921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002753"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.661336+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:24:00.661336+0000","last_peered":"2024-09-13T14:24:00.661336+0000","last_clean":"2024-09-13T14:24:00.661336+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:24:00.661336+0000","last_undegraded":"2024-09-13T14:24:00.661336+0000","last_fullsized":"2024-09-13T14:24:00.661336+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644047+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:23:59.644047+0000","last_peered":"2024-09-13T14:23:59.644047+0000","last_clean":"2024-09-13T14:23:59.644047+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:23:59.644047+0000","last_undegraded":"2024-09-13T14:23:59.644047+0000","last_fullsized":"2024-09-13T14:23:59.644047+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653974+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:24:00.653974+0000","last_peered":"2024-09-13T14:24:00.653974+0000","last_clean":"2024-09-13T14:24:00.653974+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:24:00.653974+0000","last_undegraded":"2024-09-13T14:24:00.653974+0000","last_fullsized":"2024-09-13T14:24:00.653974+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655023+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:24:00.655023+0000","last_peered":"2024-09-13T14:24:00.655023+0000","last_clean":"2024-09-13T14:24:00.655023+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:24:00.655023+0000","last_undegraded":"2024-09-13T14:24:00.655023+0000","last_fullsized":"2024-09-13T14:24:00.655023+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657045+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:24:00.657045+0000","last_peered":"2024-09-13T14:24:00.657045+0000","last_clean":"2024-09-13T14:24:00.657045+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:24:00.657045+0000","last_undegraded":"2024-09-13T14:24:00.657045+0000","last_fullsized":"2024-09-13T14:24:00.657045+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643390+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:23:59.643390+0000","last_peered":"2024-09-13T14:23:59.643390+0000","last_clean":"2024-09-13T14:23:59.643390+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:23:59.643390+0000","last_undegraded":"2024-09-13T14:23:59.643390+0000","last_fullsized":"2024-09-13T14:23:59.643390+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.651402+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:24:00.651402+0000","last_peered":"2024-09-13T14:24:00.651402+0000","last_clean":"2024-09-13T14:24:00.651402+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:24:00.651402+0000","last_undegraded":"2024-09-13T14:24:00.651402+0000","last_fullsized":"2024-09-13T14:24:00.651402+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":74,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639203+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:23:59.639203+0000","last_peered":"2024-09-13T14:23:59.639203+0000","last_clean":"2024-09-13T14:23:59.639203+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:23:59.639203+0000","last_undegraded":"2024-09-13T14:23:59.639203+0000","last_fullsized":"2024-09-13T14:23:59.639203+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644810+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:23:59.644810+0000","last_peered":"2024-09-13T14:23:59.644810+0000","last_clean":"2024-09-13T14:23:59.644810+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:23:59.644810+0000","last_undegraded":"2024-09-13T14:23:59.644810+0000","last_fullsized":"2024-09-13T14:23:59.644810+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643962+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:23:59.643962+0000","last_peered":"2024-09-13T14:23:59.643962+0000","last_clean":"2024-09-13T14:23:59.643962+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:23:59.643962+0000","last_undegraded":"2024-09-13T14:23:59.643962+0000","last_fullsized":"2024-09-13T14:23:59.643962+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645107+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:23:59.645107+0000","last_peered":"2024-09-13T14:23:59.645107+0000","last_clean":"2024-09-13T14:23:59.645107+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:23:59.645107+0000","last_undegraded":"2024-09-13T14:23:59.645107+0000","last_fullsized":"2024-09-13T14:23:59.645107+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652215+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:24:00.652215+0000","last_peered":"2024-09-13T14:24:00.652215+0000","last_clean":"2024-09-13T14:24:00.652215+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:24:00.652215+0000","last_undegraded":"2024-09-13T14:24:00.652215+0000","last_fullsized":"2024-09-13T14:24:00.652215+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":50,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645477+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:23:59.645477+0000","last_peered":"2024-09-13T14:23:59.645477+0000","last_clean":"2024-09-13T14:23:59.645477+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:23:59.645477+0000","last_undegraded":"2024-09-13T14:23:59.645477+0000","last_fullsized":"2024-09-13T14:23:59.645477+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640940+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:23:59.640940+0000","last_peered":"2024-09-13T14:23:59.640940+0000","last_clean":"2024-09-13T14:23:59.640940+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:23:59.640940+0000","last_undegraded":"2024-09-13T14:23:59.640940+0000","last_fullsized":"2024-09-13T14:23:59.640940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654168+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:24:00.654168+0000","last_peered":"2024-09-13T14:24:00.654168+0000","last_clean":"2024-09-13T14:24:00.654168+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:24:00.654168+0000","last_undegraded":"2024-09-13T14:24:00.654168+0000","last_fullsized":"2024-09-13T14:24:00.654168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643366+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:24:00.643366+0000","last_peered":"2024-09-13T14:24:00.643366+0000","last_clean":"2024-09-13T14:24:00.643366+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:24:00.643366+0000","last_undegraded":"2024-09-13T14:24:00.643366+0000","last_fullsized":"2024-09-13T14:24:00.643366+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.658797+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:24:00.658797+0000","last_peered":"2024-09-13T14:24:00.658797+0000","last_clean":"2024-09-13T14:24:00.658797+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:24:00.658797+0000","last_undegraded":"2024-09-13T14:24:00.658797+0000","last_fullsized":"2024-09-13T14:24:00.658797+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653246+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:24:00.653246+0000","last_peered":"2024-09-13T14:24:00.653246+0000","last_clean":"2024-09-13T14:24:00.653246+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:24:00.653246+0000","last_undegraded":"2024-09-13T14:24:00.653246+0000","last_fullsized":"2024-09-13T14:24:00.653246+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640237+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:23:59.640237+0000","last_peered":"2024-09-13T14:23:59.640237+0000","last_clean":"2024-09-13T14:23:59.640237+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:23:59.640237+0000","last_undegraded":"2024-09-13T14:23:59.640237+0000","last_fullsized":"2024-09-13T14:23:59.640237+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641057+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:23:59.641057+0000","last_peered":"2024-09-13T14:23:59.641057+0000","last_clean":"2024-09-13T14:23:59.641057+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:23:59.641057+0000","last_undegraded":"2024-09-13T14:23:59.641057+0000","last_fullsized":"2024-09-13T14:23:59.641057+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.642303+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:23:59.642303+0000","last_peered":"2024-09-13T14:23:59.642303+0000","last_clean":"2024-09-13T14:23:59.642303+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:23:59.642303+0000","last_undegraded":"2024-09-13T14:23:59.642303+0000","last_fullsized":"2024-09-13T14:23:59.642303+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640514+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:23:59.640514+0000","last_peered":"2024-09-13T14:23:59.640514+0000","last_clean":"2024-09-13T14:23:59.640514+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:23:59.640514+0000","last_undegraded":"2024-09-13T14:23:59.640514+0000","last_fullsized":"2024-09-13T14:23:59.640514+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652901+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:24:00.652901+0000","last_peered":"2024-09-13T14:24:00.652901+0000","last_clean":"2024-09-13T14:24:00.652901+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:24:00.652901+0000","last_undegraded":"2024-09-13T14:24:00.652901+0000","last_fullsized":"2024-09-13T14:24:00.652901+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654853+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:24:00.654853+0000","last_peered":"2024-09-13T14:24:00.654853+0000","last_clean":"2024-09-13T14:24:00.654853+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:24:00.654853+0000","last_undegraded":"2024-09-13T14:24:00.654853+0000","last_fullsized":"2024-09-13T14:24:00.654853+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":183,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644941+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:23:59.644941+0000","last_peered":"2024-09-13T14:23:59.644941+0000","last_clean":"2024-09-13T14:23:59.644941+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:23:59.644941+0000","last_undegraded":"2024-09-13T14:23:59.644941+0000","last_fullsized":"2024-09-13T14:23:59.644941+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":667,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644429+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:23:59.644429+0000","last_peered":"2024-09-13T14:23:59.644429+0000","last_clean":"2024-09-13T14:23:59.644429+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:23:59.644429+0000","last_undegraded":"2024-09-13T14:23:59.644429+0000","last_fullsized":"2024-09-13T14:23:59.644429+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653538+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:24:00.653538+0000","last_peered":"2024-09-13T14:24:00.653538+0000","last_clean":"2024-09-13T14:24:00.653538+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:24:00.653538+0000","last_undegraded":"2024-09-13T14:24:00.653538+0000","last_fullsized":"2024-09-13T14:24:00.653538+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641183+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:23:59.641183+0000","last_peered":"2024-09-13T14:23:59.641183+0000","last_clean":"2024-09-13T14:23:59.641183+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:23:59.641183+0000","last_undegraded":"2024-09-13T14:23:59.641183+0000","last_fullsized":"2024-09-13T14:23:59.641183+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":475,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644775+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:23:59.644775+0000","last_peered":"2024-09-13T14:23:59.644775+0000","last_clean":"2024-09-13T14:23:59.644775+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:23:59.644775+0000","last_undegraded":"2024-09-13T14:23:59.644775+0000","last_fullsized":"2024-09-13T14:23:59.644775+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653792+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:24:00.653792+0000","last_peered":"2024-09-13T14:24:00.653792+0000","last_clean":"2024-09-13T14:24:00.653792+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:24:00.653792+0000","last_undegraded":"2024-09-13T14:24:00.653792+0000","last_fullsized":"2024-09-13T14:24:00.653792+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.645745+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:24:00.645745+0000","last_peered":"2024-09-13T14:24:00.645745+0000","last_clean":"2024-09-13T14:24:00.645745+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:24:00.645745+0000","last_undegraded":"2024-09-13T14:24:00.645745+0000","last_fullsized":"2024-09-13T14:24:00.645745+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":494,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652899+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:24:00.652899+0000","last_peered":"2024-09-13T14:24:00.652899+0000","last_clean":"2024-09-13T14:24:00.652899+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:24:00.652899+0000","last_undegraded":"2024-09-13T14:24:00.652899+0000","last_fullsized":"2024-09-13T14:24:00.652899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643816+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:24:00.643816+0000","last_peered":"2024-09-13T14:24:00.643816+0000","last_clean":"2024-09-13T14:24:00.643816+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:24:00.643816+0000","last_undegraded":"2024-09-13T14:24:00.643816+0000","last_fullsized":"2024-09-13T14:24:00.643816+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.640811+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:24:00.640811+0000","last_peered":"2024-09-13T14:24:00.640811+0000","last_clean":"2024-09-13T14:24:00.640811+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:24:00.640811+0000","last_undegraded":"2024-09-13T14:24:00.640811+0000","last_fullsized":"2024-09-13T14:24:00.640811+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":716,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645356+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:23:59.645356+0000","last_peered":"2024-09-13T14:23:59.645356+0000","last_clean":"2024-09-13T14:23:59.645356+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:23:59.645356+0000","last_undegraded":"2024-09-13T14:23:59.645356+0000","last_fullsized":"2024-09-13T14:23:59.645356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641249+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:23:59.641249+0000","last_peered":"2024-09-13T14:23:59.641249+0000","last_clean":"2024-09-13T14:23:59.641249+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:23:59.641249+0000","last_undegraded":"2024-09-13T14:23:59.641249+0000","last_fullsized":"2024-09-13T14:23:59.641249+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655745+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:24:00.655745+0000","last_peered":"2024-09-13T14:24:00.655745+0000","last_clean":"2024-09-13T14:24:00.655745+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:24:00.655745+0000","last_undegraded":"2024-09-13T14:24:00.655745+0000","last_fullsized":"2024-09-13T14:24:00.655745+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652864+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:24:00.652864+0000","last_peered":"2024-09-13T14:24:00.652864+0000","last_clean":"2024-09-13T14:24:00.652864+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:24:00.652864+0000","last_undegraded":"2024-09-13T14:24:00.652864+0000","last_fullsized":"2024-09-13T14:24:00.652864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":665,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643732+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:24:00.643732+0000","last_peered":"2024-09-13T14:24:00.643732+0000","last_clean":"2024-09-13T14:24:00.643732+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:24:00.643732+0000","last_undegraded":"2024-09-13T14:24:00.643732+0000","last_fullsized":"2024-09-13T14:24:00.643732+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642248+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:24:00.642248+0000","last_peered":"2024-09-13T14:24:00.642248+0000","last_clean":"2024-09-13T14:24:00.642248+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:24:00.642248+0000","last_undegraded":"2024-09-13T14:24:00.642248+0000","last_fullsized":"2024-09-13T14:24:00.642248+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643629+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:24:00.643629+0000","last_peered":"2024-09-13T14:24:00.643629+0000","last_clean":"2024-09-13T14:24:00.643629+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:24:00.643629+0000","last_undegraded":"2024-09-13T14:24:00.643629+0000","last_fullsized":"2024-09-13T14:24:00.643629+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":412,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645552+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:23:59.645552+0000","last_peered":"2024-09-13T14:23:59.645552+0000","last_clean":"2024-09-13T14:23:59.645552+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:23:59.645552+0000","last_undegraded":"2024-09-13T14:23:59.645552+0000","last_fullsized":"2024-09-13T14:23:59.645552+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643279+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:24:00.643279+0000","last_peered":"2024-09-13T14:24:00.643279+0000","last_clean":"2024-09-13T14:24:00.643279+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:24:00.643279+0000","last_undegraded":"2024-09-13T14:24:00.643279+0000","last_fullsized":"2024-09-13T14:24:00.643279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659609+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:24:00.659609+0000","last_peered":"2024-09-13T14:24:00.659609+0000","last_clean":"2024-09-13T14:24:00.659609+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:24:00.659609+0000","last_undegraded":"2024-09-13T14:24:00.659609+0000","last_fullsized":"2024-09-13T14:24:00.659609+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":399,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642631+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:24:00.642631+0000","last_peered":"2024-09-13T14:24:00.642631+0000","last_clean":"2024-09-13T14:24:00.642631+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:24:00.642631+0000","last_undegraded":"2024-09-13T14:24:00.642631+0000","last_fullsized":"2024-09-13T14:24:00.642631+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652559+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:24:00.652559+0000","last_peered":"2024-09-13T14:24:00.652559+0000","last_clean":"2024-09-13T14:24:00.652559+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:24:00.652559+0000","last_undegraded":"2024-09-13T14:24:00.652559+0000","last_fullsized":"2024-09-13T14:24:00.652559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643647+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:24:00.643647+0000","last_peered":"2024-09-13T14:24:00.643647+0000","last_clean":"2024-09-13T14:24:00.643647+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:24:00.643647+0000","last_undegraded":"2024-09-13T14:24:00.643647+0000","last_fullsized":"2024-09-13T14:24:00.643647+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":520,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641454+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:23:59.641454+0000","last_peered":"2024-09-13T14:23:59.641454+0000","last_clean":"2024-09-13T14:23:59.641454+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:23:59.641454+0000","last_undegraded":"2024-09-13T14:23:59.641454+0000","last_fullsized":"2024-09-13T14:23:59.641454+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639117+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:23:59.639117+0000","last_peered":"2024-09-13T14:23:59.639117+0000","last_clean":"2024-09-13T14:23:59.639117+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:23:59.639117+0000","last_undegraded":"2024-09-13T14:23:59.639117+0000","last_fullsized":"2024-09-13T14:23:59.639117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641076+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:23:59.641076+0000","last_peered":"2024-09-13T14:23:59.641076+0000","last_clean":"2024-09-13T14:23:59.641076+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:23:59.641076+0000","last_undegraded":"2024-09-13T14:23:59.641076+0000","last_fullsized":"2024-09-13T14:23:59.641076+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":72,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639435+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:23:59.639435+0000","last_peered":"2024-09-13T14:23:59.639435+0000","last_clean":"2024-09-13T14:23:59.639435+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:23:59.639435+0000","last_undegraded":"2024-09-13T14:23:59.639435+0000","last_fullsized":"2024-09-13T14:23:59.639435+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642056+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:24:00.642056+0000","last_peered":"2024-09-13T14:24:00.642056+0000","last_clean":"2024-09-13T14:24:00.642056+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:24:00.642056+0000","last_undegraded":"2024-09-13T14:24:00.642056+0000","last_fullsized":"2024-09-13T14:24:00.642056+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.650848+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:24:00.650848+0000","last_peered":"2024-09-13T14:24:00.650848+0000","last_clean":"2024-09-13T14:24:00.650848+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:24:00.650848+0000","last_undegraded":"2024-09-13T14:24:00.650848+0000","last_fullsized":"2024-09-13T14:24:00.650848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644269+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:23:59.644269+0000","last_peered":"2024-09-13T14:23:59.644269+0000","last_clean":"2024-09-13T14:23:59.644269+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:23:59.644269+0000","last_undegraded":"2024-09-13T14:23:59.644269+0000","last_fullsized":"2024-09-13T14:23:59.644269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657819+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:24:00.657819+0000","last_peered":"2024-09-13T14:24:00.657819+0000","last_clean":"2024-09-13T14:24:00.657819+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:24:00.657819+0000","last_undegraded":"2024-09-13T14:24:00.657819+0000","last_fullsized":"2024-09-13T14:24:00.657819+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655260+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:24:00.655260+0000","last_peered":"2024-09-13T14:24:00.655260+0000","last_clean":"2024-09-13T14:24:00.655260+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:24:00.655260+0000","last_undegraded":"2024-09-13T14:24:00.655260+0000","last_fullsized":"2024-09-13T14:24:00.655260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653009+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:24:00.653009+0000","last_peered":"2024-09-13T14:24:00.653009+0000","last_clean":"2024-09-13T14:24:00.653009+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:24:00.653009+0000","last_undegraded":"2024-09-13T14:24:00.653009+0000","last_fullsized":"2024-09-13T14:24:00.653009+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641367+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:23:59.641367+0000","last_peered":"2024-09-13T14:23:59.641367+0000","last_clean":"2024-09-13T14:23:59.641367+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:23:59.641367+0000","last_undegraded":"2024-09-13T14:23:59.641367+0000","last_fullsized":"2024-09-13T14:23:59.641367+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652277+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:24:00.652277+0000","last_peered":"2024-09-13T14:24:00.652277+0000","last_clean":"2024-09-13T14:24:00.652277+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:24:00.652277+0000","last_undegraded":"2024-09-13T14:24:00.652277+0000","last_fullsized":"2024-09-13T14:24:00.652277+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657178+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:24:00.657178+0000","last_peered":"2024-09-13T14:24:00.657178+0000","last_clean":"2024-09-13T14:24:00.657178+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:24:00.657178+0000","last_undegraded":"2024-09-13T14:24:00.657178+0000","last_fullsized":"2024-09-13T14:24:00.657178+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654244+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:24:00.654244+0000","last_peered":"2024-09-13T14:24:00.654244+0000","last_clean":"2024-09-13T14:24:00.654244+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:24:00.654244+0000","last_undegraded":"2024-09-13T14:24:00.654244+0000","last_fullsized":"2024-09-13T14:24:00.654244+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653972+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:24:00.653972+0000","last_peered":"2024-09-13T14:24:00.653972+0000","last_clean":"2024-09-13T14:24:00.653972+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:24:00.653972+0000","last_undegraded":"2024-09-13T14:24:00.653972+0000","last_fullsized":"2024-09-13T14:24:00.653972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":85,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639960+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:23:59.639960+0000","last_peered":"2024-09-13T14:23:59.639960+0000","last_clean":"2024-09-13T14:23:59.639960+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:23:59.639960+0000","last_undegraded":"2024-09-13T14:23:59.639960+0000","last_fullsized":"2024-09-13T14:23:59.639960+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654744+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:24:00.654744+0000","last_peered":"2024-09-13T14:24:00.654744+0000","last_clean":"2024-09-13T14:24:00.654744+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:24:00.654744+0000","last_undegraded":"2024-09-13T14:24:00.654744+0000","last_fullsized":"2024-09-13T14:24:00.654744+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":58,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643227+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:24:00.643227+0000","last_peered":"2024-09-13T14:24:00.643227+0000","last_clean":"2024-09-13T14:24:00.643227+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:24:00.643227+0000","last_undegraded":"2024-09-13T14:24:00.643227+0000","last_fullsized":"2024-09-13T14:24:00.643227+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652248+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:24:00.652248+0000","last_peered":"2024-09-13T14:24:00.652248+0000","last_clean":"2024-09-13T14:24:00.652248+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:24:00.652248+0000","last_undegraded":"2024-09-13T14:24:00.652248+0000","last_fullsized":"2024-09-13T14:24:00.652248+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.646313+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:23:59.646313+0000","last_peered":"2024-09-13T14:23:59.646313+0000","last_clean":"2024-09-13T14:23:59.646313+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:23:59.646313+0000","last_undegraded":"2024-09-13T14:23:59.646313+0000","last_fullsized":"2024-09-13T14:23:59.646313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652085+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:24:00.652085+0000","last_peered":"2024-09-13T14:24:00.652085+0000","last_clean":"2024-09-13T14:24:00.652085+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:24:00.652085+0000","last_undegraded":"2024-09-13T14:24:00.652085+0000","last_fullsized":"2024-09-13T14:24:00.652085+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":51,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645618+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:23:59.645618+0000","last_peered":"2024-09-13T14:23:59.645618+0000","last_clean":"2024-09-13T14:23:59.645618+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:23:59.645618+0000","last_undegraded":"2024-09-13T14:23:59.645618+0000","last_fullsized":"2024-09-13T14:23:59.645618+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659564+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:24:00.659564+0000","last_peered":"2024-09-13T14:24:00.659564+0000","last_clean":"2024-09-13T14:24:00.659564+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:24:00.659564+0000","last_undegraded":"2024-09-13T14:24:00.659564+0000","last_fullsized":"2024-09-13T14:24:00.659564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":75,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654100+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:24:00.654100+0000","last_peered":"2024-09-13T14:24:00.654100+0000","last_clean":"2024-09-13T14:24:00.654100+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:24:00.654100+0000","last_undegraded":"2024-09-13T14:24:00.654100+0000","last_fullsized":"2024-09-13T14:24:00.654100+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639012+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:23:59.639012+0000","last_peered":"2024-09-13T14:23:59.639012+0000","last_clean":"2024-09-13T14:23:59.639012+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:23:59.639012+0000","last_undegraded":"2024-09-13T14:23:59.639012+0000","last_fullsized":"2024-09-13T14:23:59.639012+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.644014+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:24:00.644014+0000","last_peered":"2024-09-13T14:24:00.644014+0000","last_clean":"2024-09-13T14:24:00.644014+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:24:00.644014+0000","last_undegraded":"2024-09-13T14:24:00.644014+0000","last_fullsized":"2024-09-13T14:24:00.644014+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397667,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307316,"kb_used_data":5792,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437836,"statfs":{"total":95995035648,"available":95680344064,"internally_reserved":0,"allocated":5931008,"data_stored":5374887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593895,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299044,"kb_used_data":1680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446108,"statfs":{"total":95995035648,"available":95688814592,"internally_reserved":0,"allocated":1720320,"data_stored":1179889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790125,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298456,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446696,"statfs":{"total":95995035648,"available":95689416704,"internally_reserved":0,"allocated":1118208,"data_stored":585284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986353,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298468,"kb_used_data":1112,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446684,"statfs":{"total":95995035648,"available":95689404416,"internally_reserved":0,"allocated":1138688,"data_stored":590145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696911,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1092,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1118208,"data_stored":585357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":28109899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860438,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1096,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1122304,"data_stored":585879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":32431948},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842023964,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36880,"kb_used_data":5188,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708272,"statfs":{"total":95995035648,"available":95957270528,"internally_reserved":0,"allocated":5312512,"data_stored":4779582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":32432567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187499,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":5768,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":5906432,"data_stored":5370224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32432459},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:25:07.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:25:08.045 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:08 smithi017 ceph-mon[103001]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:08 smithi017 ceph-mon[107556]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:08 smithi189 ceph-mon[90992]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:09.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:09 smithi017 ceph-mon[103001]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:09 smithi017 ceph-mon[107556]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:09 smithi189 ceph-mon[90992]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:09.872 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:25:09.873 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:25:10.454 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":232,"stamp":"2024-09-13T14:25:09.725949+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6304,"num_read_kb":13770,"num_write":1056,"num_write_kb":14251,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1132,"ondisk_log_size":1132,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338996,"kb_used_data":22820,"kb_used_omap":35,"kb_used_meta":1315996,"kb_avail":748622220,"statfs":{"total":767960285184,"available":766589153280,"internally_reserved":0,"allocated":23367680,"data_stored":19051247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1347580921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002859"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.661336+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:24:00.661336+0000","last_peered":"2024-09-13T14:24:00.661336+0000","last_clean":"2024-09-13T14:24:00.661336+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:24:00.661336+0000","last_undegraded":"2024-09-13T14:24:00.661336+0000","last_fullsized":"2024-09-13T14:24:00.661336+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644047+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:23:59.644047+0000","last_peered":"2024-09-13T14:23:59.644047+0000","last_clean":"2024-09-13T14:23:59.644047+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:23:59.644047+0000","last_undegraded":"2024-09-13T14:23:59.644047+0000","last_fullsized":"2024-09-13T14:23:59.644047+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653974+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:24:00.653974+0000","last_peered":"2024-09-13T14:24:00.653974+0000","last_clean":"2024-09-13T14:24:00.653974+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:24:00.653974+0000","last_undegraded":"2024-09-13T14:24:00.653974+0000","last_fullsized":"2024-09-13T14:24:00.653974+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655023+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:24:00.655023+0000","last_peered":"2024-09-13T14:24:00.655023+0000","last_clean":"2024-09-13T14:24:00.655023+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:24:00.655023+0000","last_undegraded":"2024-09-13T14:24:00.655023+0000","last_fullsized":"2024-09-13T14:24:00.655023+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657045+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:24:00.657045+0000","last_peered":"2024-09-13T14:24:00.657045+0000","last_clean":"2024-09-13T14:24:00.657045+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:24:00.657045+0000","last_undegraded":"2024-09-13T14:24:00.657045+0000","last_fullsized":"2024-09-13T14:24:00.657045+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643390+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:23:59.643390+0000","last_peered":"2024-09-13T14:23:59.643390+0000","last_clean":"2024-09-13T14:23:59.643390+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:23:59.643390+0000","last_undegraded":"2024-09-13T14:23:59.643390+0000","last_fullsized":"2024-09-13T14:23:59.643390+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.651402+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:24:00.651402+0000","last_peered":"2024-09-13T14:24:00.651402+0000","last_clean":"2024-09-13T14:24:00.651402+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:24:00.651402+0000","last_undegraded":"2024-09-13T14:24:00.651402+0000","last_fullsized":"2024-09-13T14:24:00.651402+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":74,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639203+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:23:59.639203+0000","last_peered":"2024-09-13T14:23:59.639203+0000","last_clean":"2024-09-13T14:23:59.639203+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:23:59.639203+0000","last_undegraded":"2024-09-13T14:23:59.639203+0000","last_fullsized":"2024-09-13T14:23:59.639203+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644810+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:23:59.644810+0000","last_peered":"2024-09-13T14:23:59.644810+0000","last_clean":"2024-09-13T14:23:59.644810+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:23:59.644810+0000","last_undegraded":"2024-09-13T14:23:59.644810+0000","last_fullsized":"2024-09-13T14:23:59.644810+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.643962+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:23:59.643962+0000","last_peered":"2024-09-13T14:23:59.643962+0000","last_clean":"2024-09-13T14:23:59.643962+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:23:59.643962+0000","last_undegraded":"2024-09-13T14:23:59.643962+0000","last_fullsized":"2024-09-13T14:23:59.643962+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645107+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:23:59.645107+0000","last_peered":"2024-09-13T14:23:59.645107+0000","last_clean":"2024-09-13T14:23:59.645107+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:23:59.645107+0000","last_undegraded":"2024-09-13T14:23:59.645107+0000","last_fullsized":"2024-09-13T14:23:59.645107+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652215+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:24:00.652215+0000","last_peered":"2024-09-13T14:24:00.652215+0000","last_clean":"2024-09-13T14:24:00.652215+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:24:00.652215+0000","last_undegraded":"2024-09-13T14:24:00.652215+0000","last_fullsized":"2024-09-13T14:24:00.652215+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":50,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645477+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:23:59.645477+0000","last_peered":"2024-09-13T14:23:59.645477+0000","last_clean":"2024-09-13T14:23:59.645477+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:23:59.645477+0000","last_undegraded":"2024-09-13T14:23:59.645477+0000","last_fullsized":"2024-09-13T14:23:59.645477+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640940+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:23:59.640940+0000","last_peered":"2024-09-13T14:23:59.640940+0000","last_clean":"2024-09-13T14:23:59.640940+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:23:59.640940+0000","last_undegraded":"2024-09-13T14:23:59.640940+0000","last_fullsized":"2024-09-13T14:23:59.640940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654168+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:24:00.654168+0000","last_peered":"2024-09-13T14:24:00.654168+0000","last_clean":"2024-09-13T14:24:00.654168+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:24:00.654168+0000","last_undegraded":"2024-09-13T14:24:00.654168+0000","last_fullsized":"2024-09-13T14:24:00.654168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643366+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:24:00.643366+0000","last_peered":"2024-09-13T14:24:00.643366+0000","last_clean":"2024-09-13T14:24:00.643366+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:24:00.643366+0000","last_undegraded":"2024-09-13T14:24:00.643366+0000","last_fullsized":"2024-09-13T14:24:00.643366+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.658797+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:24:00.658797+0000","last_peered":"2024-09-13T14:24:00.658797+0000","last_clean":"2024-09-13T14:24:00.658797+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:24:00.658797+0000","last_undegraded":"2024-09-13T14:24:00.658797+0000","last_fullsized":"2024-09-13T14:24:00.658797+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653246+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:24:00.653246+0000","last_peered":"2024-09-13T14:24:00.653246+0000","last_clean":"2024-09-13T14:24:00.653246+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:24:00.653246+0000","last_undegraded":"2024-09-13T14:24:00.653246+0000","last_fullsized":"2024-09-13T14:24:00.653246+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640237+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:23:59.640237+0000","last_peered":"2024-09-13T14:23:59.640237+0000","last_clean":"2024-09-13T14:23:59.640237+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:23:59.640237+0000","last_undegraded":"2024-09-13T14:23:59.640237+0000","last_fullsized":"2024-09-13T14:23:59.640237+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641057+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:23:59.641057+0000","last_peered":"2024-09-13T14:23:59.641057+0000","last_clean":"2024-09-13T14:23:59.641057+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:23:59.641057+0000","last_undegraded":"2024-09-13T14:23:59.641057+0000","last_fullsized":"2024-09-13T14:23:59.641057+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.642303+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:23:59.642303+0000","last_peered":"2024-09-13T14:23:59.642303+0000","last_clean":"2024-09-13T14:23:59.642303+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:23:59.642303+0000","last_undegraded":"2024-09-13T14:23:59.642303+0000","last_fullsized":"2024-09-13T14:23:59.642303+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.640514+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:23:59.640514+0000","last_peered":"2024-09-13T14:23:59.640514+0000","last_clean":"2024-09-13T14:23:59.640514+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:23:59.640514+0000","last_undegraded":"2024-09-13T14:23:59.640514+0000","last_fullsized":"2024-09-13T14:23:59.640514+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652901+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:24:00.652901+0000","last_peered":"2024-09-13T14:24:00.652901+0000","last_clean":"2024-09-13T14:24:00.652901+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:24:00.652901+0000","last_undegraded":"2024-09-13T14:24:00.652901+0000","last_fullsized":"2024-09-13T14:24:00.652901+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655282+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:24:00.655282+0000","last_peered":"2024-09-13T14:24:00.655282+0000","last_clean":"2024-09-13T14:24:00.655282+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:24:00.655282+0000","last_undegraded":"2024-09-13T14:24:00.655282+0000","last_fullsized":"2024-09-13T14:24:00.655282+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":183,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644941+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:23:59.644941+0000","last_peered":"2024-09-13T14:23:59.644941+0000","last_clean":"2024-09-13T14:23:59.644941+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:23:59.644941+0000","last_undegraded":"2024-09-13T14:23:59.644941+0000","last_fullsized":"2024-09-13T14:23:59.644941+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":667,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644429+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:23:59.644429+0000","last_peered":"2024-09-13T14:23:59.644429+0000","last_clean":"2024-09-13T14:23:59.644429+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:23:59.644429+0000","last_undegraded":"2024-09-13T14:23:59.644429+0000","last_fullsized":"2024-09-13T14:23:59.644429+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653538+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:24:00.653538+0000","last_peered":"2024-09-13T14:24:00.653538+0000","last_clean":"2024-09-13T14:24:00.653538+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:24:00.653538+0000","last_undegraded":"2024-09-13T14:24:00.653538+0000","last_fullsized":"2024-09-13T14:24:00.653538+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":48,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641183+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:23:59.641183+0000","last_peered":"2024-09-13T14:23:59.641183+0000","last_clean":"2024-09-13T14:23:59.641183+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:23:59.641183+0000","last_undegraded":"2024-09-13T14:23:59.641183+0000","last_fullsized":"2024-09-13T14:23:59.641183+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":475,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644775+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:23:59.644775+0000","last_peered":"2024-09-13T14:23:59.644775+0000","last_clean":"2024-09-13T14:23:59.644775+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:23:59.644775+0000","last_undegraded":"2024-09-13T14:23:59.644775+0000","last_fullsized":"2024-09-13T14:23:59.644775+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653792+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:24:00.653792+0000","last_peered":"2024-09-13T14:24:00.653792+0000","last_clean":"2024-09-13T14:24:00.653792+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:24:00.653792+0000","last_undegraded":"2024-09-13T14:24:00.653792+0000","last_fullsized":"2024-09-13T14:24:00.653792+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.645745+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:24:00.645745+0000","last_peered":"2024-09-13T14:24:00.645745+0000","last_clean":"2024-09-13T14:24:00.645745+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:24:00.645745+0000","last_undegraded":"2024-09-13T14:24:00.645745+0000","last_fullsized":"2024-09-13T14:24:00.645745+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":494,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652899+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:24:00.652899+0000","last_peered":"2024-09-13T14:24:00.652899+0000","last_clean":"2024-09-13T14:24:00.652899+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:24:00.652899+0000","last_undegraded":"2024-09-13T14:24:00.652899+0000","last_fullsized":"2024-09-13T14:24:00.652899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643816+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:24:00.643816+0000","last_peered":"2024-09-13T14:24:00.643816+0000","last_clean":"2024-09-13T14:24:00.643816+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:24:00.643816+0000","last_undegraded":"2024-09-13T14:24:00.643816+0000","last_fullsized":"2024-09-13T14:24:00.643816+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.640811+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:24:00.640811+0000","last_peered":"2024-09-13T14:24:00.640811+0000","last_clean":"2024-09-13T14:24:00.640811+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:24:00.640811+0000","last_undegraded":"2024-09-13T14:24:00.640811+0000","last_fullsized":"2024-09-13T14:24:00.640811+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":716,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645356+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:23:59.645356+0000","last_peered":"2024-09-13T14:23:59.645356+0000","last_clean":"2024-09-13T14:23:59.645356+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:23:59.645356+0000","last_undegraded":"2024-09-13T14:23:59.645356+0000","last_fullsized":"2024-09-13T14:23:59.645356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641249+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:23:59.641249+0000","last_peered":"2024-09-13T14:23:59.641249+0000","last_clean":"2024-09-13T14:23:59.641249+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:23:59.641249+0000","last_undegraded":"2024-09-13T14:23:59.641249+0000","last_fullsized":"2024-09-13T14:23:59.641249+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655745+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:24:00.655745+0000","last_peered":"2024-09-13T14:24:00.655745+0000","last_clean":"2024-09-13T14:24:00.655745+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:24:00.655745+0000","last_undegraded":"2024-09-13T14:24:00.655745+0000","last_fullsized":"2024-09-13T14:24:00.655745+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652864+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:24:00.652864+0000","last_peered":"2024-09-13T14:24:00.652864+0000","last_clean":"2024-09-13T14:24:00.652864+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:24:00.652864+0000","last_undegraded":"2024-09-13T14:24:00.652864+0000","last_fullsized":"2024-09-13T14:24:00.652864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":665,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643732+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:24:00.643732+0000","last_peered":"2024-09-13T14:24:00.643732+0000","last_clean":"2024-09-13T14:24:00.643732+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:24:00.643732+0000","last_undegraded":"2024-09-13T14:24:00.643732+0000","last_fullsized":"2024-09-13T14:24:00.643732+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642248+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:24:00.642248+0000","last_peered":"2024-09-13T14:24:00.642248+0000","last_clean":"2024-09-13T14:24:00.642248+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:24:00.642248+0000","last_undegraded":"2024-09-13T14:24:00.642248+0000","last_fullsized":"2024-09-13T14:24:00.642248+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643629+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:24:00.643629+0000","last_peered":"2024-09-13T14:24:00.643629+0000","last_clean":"2024-09-13T14:24:00.643629+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:24:00.643629+0000","last_undegraded":"2024-09-13T14:24:00.643629+0000","last_fullsized":"2024-09-13T14:24:00.643629+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":412,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645552+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:23:59.645552+0000","last_peered":"2024-09-13T14:23:59.645552+0000","last_clean":"2024-09-13T14:23:59.645552+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:23:59.645552+0000","last_undegraded":"2024-09-13T14:23:59.645552+0000","last_fullsized":"2024-09-13T14:23:59.645552+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643279+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:24:00.643279+0000","last_peered":"2024-09-13T14:24:00.643279+0000","last_clean":"2024-09-13T14:24:00.643279+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:24:00.643279+0000","last_undegraded":"2024-09-13T14:24:00.643279+0000","last_fullsized":"2024-09-13T14:24:00.643279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":67,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659609+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:24:00.659609+0000","last_peered":"2024-09-13T14:24:00.659609+0000","last_clean":"2024-09-13T14:24:00.659609+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:24:00.659609+0000","last_undegraded":"2024-09-13T14:24:00.659609+0000","last_fullsized":"2024-09-13T14:24:00.659609+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":399,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642631+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:24:00.642631+0000","last_peered":"2024-09-13T14:24:00.642631+0000","last_clean":"2024-09-13T14:24:00.642631+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:24:00.642631+0000","last_undegraded":"2024-09-13T14:24:00.642631+0000","last_fullsized":"2024-09-13T14:24:00.642631+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652559+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:24:00.652559+0000","last_peered":"2024-09-13T14:24:00.652559+0000","last_clean":"2024-09-13T14:24:00.652559+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:24:00.652559+0000","last_undegraded":"2024-09-13T14:24:00.652559+0000","last_fullsized":"2024-09-13T14:24:00.652559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643647+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:24:00.643647+0000","last_peered":"2024-09-13T14:24:00.643647+0000","last_clean":"2024-09-13T14:24:00.643647+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:24:00.643647+0000","last_undegraded":"2024-09-13T14:24:00.643647+0000","last_fullsized":"2024-09-13T14:24:00.643647+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":520,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641454+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:23:59.641454+0000","last_peered":"2024-09-13T14:23:59.641454+0000","last_clean":"2024-09-13T14:23:59.641454+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:23:59.641454+0000","last_undegraded":"2024-09-13T14:23:59.641454+0000","last_fullsized":"2024-09-13T14:23:59.641454+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639117+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:23:59.639117+0000","last_peered":"2024-09-13T14:23:59.639117+0000","last_clean":"2024-09-13T14:23:59.639117+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:23:59.639117+0000","last_undegraded":"2024-09-13T14:23:59.639117+0000","last_fullsized":"2024-09-13T14:23:59.639117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641076+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:23:59.641076+0000","last_peered":"2024-09-13T14:23:59.641076+0000","last_clean":"2024-09-13T14:23:59.641076+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:23:59.641076+0000","last_undegraded":"2024-09-13T14:23:59.641076+0000","last_fullsized":"2024-09-13T14:23:59.641076+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":72,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639435+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:23:59.639435+0000","last_peered":"2024-09-13T14:23:59.639435+0000","last_clean":"2024-09-13T14:23:59.639435+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:23:59.639435+0000","last_undegraded":"2024-09-13T14:23:59.639435+0000","last_fullsized":"2024-09-13T14:23:59.639435+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.642056+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:24:00.642056+0000","last_peered":"2024-09-13T14:24:00.642056+0000","last_clean":"2024-09-13T14:24:00.642056+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:24:00.642056+0000","last_undegraded":"2024-09-13T14:24:00.642056+0000","last_fullsized":"2024-09-13T14:24:00.642056+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.650848+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:24:00.650848+0000","last_peered":"2024-09-13T14:24:00.650848+0000","last_clean":"2024-09-13T14:24:00.650848+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:24:00.650848+0000","last_undegraded":"2024-09-13T14:24:00.650848+0000","last_fullsized":"2024-09-13T14:24:00.650848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.644269+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:23:59.644269+0000","last_peered":"2024-09-13T14:23:59.644269+0000","last_clean":"2024-09-13T14:23:59.644269+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:23:59.644269+0000","last_undegraded":"2024-09-13T14:23:59.644269+0000","last_fullsized":"2024-09-13T14:23:59.644269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":24,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657819+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:24:00.657819+0000","last_peered":"2024-09-13T14:24:00.657819+0000","last_clean":"2024-09-13T14:24:00.657819+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:24:00.657819+0000","last_undegraded":"2024-09-13T14:24:00.657819+0000","last_fullsized":"2024-09-13T14:24:00.657819+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":65,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.655260+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:24:00.655260+0000","last_peered":"2024-09-13T14:24:00.655260+0000","last_clean":"2024-09-13T14:24:00.655260+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:24:00.655260+0000","last_undegraded":"2024-09-13T14:24:00.655260+0000","last_fullsized":"2024-09-13T14:24:00.655260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653009+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:24:00.653009+0000","last_peered":"2024-09-13T14:24:00.653009+0000","last_clean":"2024-09-13T14:24:00.653009+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:24:00.653009+0000","last_undegraded":"2024-09-13T14:24:00.653009+0000","last_fullsized":"2024-09-13T14:24:00.653009+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.641367+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:23:59.641367+0000","last_peered":"2024-09-13T14:23:59.641367+0000","last_clean":"2024-09-13T14:23:59.641367+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:23:59.641367+0000","last_undegraded":"2024-09-13T14:23:59.641367+0000","last_fullsized":"2024-09-13T14:23:59.641367+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652277+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:24:00.652277+0000","last_peered":"2024-09-13T14:24:00.652277+0000","last_clean":"2024-09-13T14:24:00.652277+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:24:00.652277+0000","last_undegraded":"2024-09-13T14:24:00.652277+0000","last_fullsized":"2024-09-13T14:24:00.652277+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.657178+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:24:00.657178+0000","last_peered":"2024-09-13T14:24:00.657178+0000","last_clean":"2024-09-13T14:24:00.657178+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:24:00.657178+0000","last_undegraded":"2024-09-13T14:24:00.657178+0000","last_fullsized":"2024-09-13T14:24:00.657178+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654244+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:24:00.654244+0000","last_peered":"2024-09-13T14:24:00.654244+0000","last_clean":"2024-09-13T14:24:00.654244+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:24:00.654244+0000","last_undegraded":"2024-09-13T14:24:00.654244+0000","last_fullsized":"2024-09-13T14:24:00.654244+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.653972+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:24:00.653972+0000","last_peered":"2024-09-13T14:24:00.653972+0000","last_clean":"2024-09-13T14:24:00.653972+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:24:00.653972+0000","last_undegraded":"2024-09-13T14:24:00.653972+0000","last_fullsized":"2024-09-13T14:24:00.653972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":85,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639960+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:23:59.639960+0000","last_peered":"2024-09-13T14:23:59.639960+0000","last_clean":"2024-09-13T14:23:59.639960+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:23:59.639960+0000","last_undegraded":"2024-09-13T14:23:59.639960+0000","last_fullsized":"2024-09-13T14:23:59.639960+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654744+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:24:00.654744+0000","last_peered":"2024-09-13T14:24:00.654744+0000","last_clean":"2024-09-13T14:24:00.654744+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:24:00.654744+0000","last_undegraded":"2024-09-13T14:24:00.654744+0000","last_fullsized":"2024-09-13T14:24:00.654744+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":58,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.643227+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:24:00.643227+0000","last_peered":"2024-09-13T14:24:00.643227+0000","last_clean":"2024-09-13T14:24:00.643227+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:24:00.643227+0000","last_undegraded":"2024-09-13T14:24:00.643227+0000","last_fullsized":"2024-09-13T14:24:00.643227+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652248+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:24:00.652248+0000","last_peered":"2024-09-13T14:24:00.652248+0000","last_clean":"2024-09-13T14:24:00.652248+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:24:00.652248+0000","last_undegraded":"2024-09-13T14:24:00.652248+0000","last_fullsized":"2024-09-13T14:24:00.652248+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":68,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.646313+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:23:59.646313+0000","last_peered":"2024-09-13T14:23:59.646313+0000","last_clean":"2024-09-13T14:23:59.646313+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:23:59.646313+0000","last_undegraded":"2024-09-13T14:23:59.646313+0000","last_fullsized":"2024-09-13T14:23:59.646313+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.652085+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:24:00.652085+0000","last_peered":"2024-09-13T14:24:00.652085+0000","last_clean":"2024-09-13T14:24:00.652085+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:24:00.652085+0000","last_undegraded":"2024-09-13T14:24:00.652085+0000","last_fullsized":"2024-09-13T14:24:00.652085+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":51,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.645618+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:23:59.645618+0000","last_peered":"2024-09-13T14:23:59.645618+0000","last_clean":"2024-09-13T14:23:59.645618+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:23:59.645618+0000","last_undegraded":"2024-09-13T14:23:59.645618+0000","last_fullsized":"2024-09-13T14:23:59.645618+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.659564+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:24:00.659564+0000","last_peered":"2024-09-13T14:24:00.659564+0000","last_clean":"2024-09-13T14:24:00.659564+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:24:00.659564+0000","last_undegraded":"2024-09-13T14:24:00.659564+0000","last_fullsized":"2024-09-13T14:24:00.659564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":75,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.654100+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:24:00.654100+0000","last_peered":"2024-09-13T14:24:00.654100+0000","last_clean":"2024-09-13T14:24:00.654100+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:24:00.654100+0000","last_undegraded":"2024-09-13T14:24:00.654100+0000","last_fullsized":"2024-09-13T14:24:00.654100+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":76,"reported_epoch":101,"state":"active+clean","last_fresh":"2024-09-13T14:23:59.639012+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:23:59.639012+0000","last_peered":"2024-09-13T14:23:59.639012+0000","last_clean":"2024-09-13T14:23:59.639012+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:23:59.639012+0000","last_undegraded":"2024-09-13T14:23:59.639012+0000","last_fullsized":"2024-09-13T14:23:59.639012+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":69,"reported_epoch":102,"state":"active+clean","last_fresh":"2024-09-13T14:24:00.644014+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:24:00.644014+0000","last_peered":"2024-09-13T14:24:00.644014+0000","last_clean":"2024-09-13T14:24:00.644014+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:24:00.644014+0000","last_undegraded":"2024-09-13T14:24:00.644014+0000","last_fullsized":"2024-09-13T14:24:00.644014+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397668,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307316,"kb_used_data":5792,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93437836,"statfs":{"total":95995035648,"available":95680344064,"internally_reserved":0,"allocated":5931008,"data_stored":5374887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593896,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299044,"kb_used_data":1680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446108,"statfs":{"total":95995035648,"available":95688814592,"internally_reserved":0,"allocated":1720320,"data_stored":1179889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790126,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298456,"kb_used_data":1092,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446696,"statfs":{"total":95995035648,"available":95689416704,"internally_reserved":0,"allocated":1118208,"data_stored":585284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986354,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298468,"kb_used_data":1112,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446684,"statfs":{"total":95995035648,"available":95689404416,"internally_reserved":0,"allocated":1138688,"data_stored":590145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696912,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1092,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1118208,"data_stored":585357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":28109899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860438,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32796,"kb_used_data":1096,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712356,"statfs":{"total":95995035648,"available":95961452544,"internally_reserved":0,"allocated":1122304,"data_stored":585879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":32431948},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842023965,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36880,"kb_used_data":5188,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708272,"statfs":{"total":95995035648,"available":95957270528,"internally_reserved":0,"allocated":5312512,"data_stored":4779582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":32432567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187500,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37468,"kb_used_data":5768,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707684,"statfs":{"total":95995035648,"available":95956668416,"internally_reserved":0,"allocated":5906432,"data_stored":5370224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32432459},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:25:10.457 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:25:10.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-13T14:25:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:10.714 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:11 smithi017 ceph-mon[103001]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:11 smithi017 ceph-mon[103001]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:11 smithi017 ceph-mon[107556]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:11 smithi017 ceph-mon[107556]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:11 smithi189 ceph-mon[90992]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:11 smithi189 ceph-mon[90992]: from='client.34977 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:25:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:12 smithi017 ceph-mon[107556]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:12 smithi017 ceph-mon[103001]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:12.577 INFO:teuthology.orchestra.run.smithi017.stdout:epoch 102 2024-09-13T14:25:12.577 INFO:teuthology.orchestra.run.smithi017.stdout:fsid 523f52e4-71d9-11ef-bceb-c7b262605968 2024-09-13T14:25:12.577 INFO:teuthology.orchestra.run.smithi017.stdout:created 2024-09-13T14:06:56.701743+0000 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:modified 2024-09-13T14:24:00.637335+0000 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:crush_version 22 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:full_ratio 0.95 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:backfillfull_ratio 0.9 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:nearfull_ratio 0.85 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:require_min_compat_client luminous 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:min_compat_client luminous 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:require_osd_release quincy 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:stretch_mode_enabled false 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 78 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-13T14:25:12.578 INFO:teuthology.orchestra.run.smithi017.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 64 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:max_osd 8 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 up in weight 1 up_from 86 up_thru 101 down_at 83 last_clean_interval [8,82) [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078] [v2:172.21.15.17:6804/59727078,v1:172.21.15.17:6805/59727078] exists,up b9a98879-a576-4dbf-80ce-247ecc208028 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 up in weight 1 up_from 91 up_thru 101 down_at 88 last_clean_interval [13,87) [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563] [v2:172.21.15.17:6812/3641293563,v1:172.21.15.17:6813/3641293563] exists,up f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 up in weight 1 up_from 96 up_thru 101 down_at 93 last_clean_interval [18,92) [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397] [v2:172.21.15.17:6820/847801397,v1:172.21.15.17:6821/847801397] exists,up eb11f41d-35bc-4180-a0f6-781c7dbe5b5b 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 up in weight 1 up_from 101 up_thru 101 down_at 98 last_clean_interval [26,97) [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290] [v2:172.21.15.17:6828/594040290,v1:172.21.15.17:6829/594040290] exists,up 76cf0d9c-0405-4601-9c2c-3cbbba55dab8 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 up in weight 1 up_from 31 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996] [v2:172.21.15.189:6802/3136403996,v1:172.21.15.189:6803/3136403996] exists,up 698eddb3-30d9-431e-9731-9c37441f4359 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 up in weight 1 up_from 37 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919] [v2:172.21.15.189:6810/379852919,v1:172.21.15.189:6811/379852919] exists,up 60f030c1-c222-4a1e-b7d6-d788f37bc5eb 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 up in weight 1 up_from 43 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256] [v2:172.21.15.189:6818/2742951256,v1:172.21.15.189:6819/2742951256] exists,up a53bc1f8-35e0-4e97-a9d1-90b318be2165 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 up in weight 1 up_from 49 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107] [v2:172.21.15.189:6826/1790039107,v1:172.21.15.189:6827/1790039107] exists,up a3fdb8ae-2c63-493d-b2ac-8009b024fa12 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.7 [1,2] 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.d [1,5] 2024-09-13T14:25:12.579 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.18 [1,2] 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/3409877178 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/3409877178 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3929495183 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3205338252 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1432283173 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/502430602 expires 2024-09-14T14:07:45.879701+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:6835/862123899 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2755559292 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3701412343 expires 2024-09-14T14:07:23.765308+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3987946356 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3469824274 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/3334536375 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2573022556 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.580 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/652586513 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/2946870083 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/2742834835 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/4035711494 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3476899836 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:6834/862123899 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1490019363 expires 2024-09-14T14:07:45.879701+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/502430602 expires 2024-09-14T14:07:45.879701+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/2902550950 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3173583319 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3527746617 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/1507399137 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3892697089 expires 2024-09-14T14:07:23.765308+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/2742834835 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.581 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/2902550950 expires 2024-09-14T14:12:23.277718+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/282666625 expires 2024-09-14T14:18:11.614839+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/2576928215 expires 2024-09-14T14:07:23.765308+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1515064414 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/4174048439 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/101070012 expires 2024-09-14T14:16:12.599650+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/2576928215 expires 2024-09-14T14:07:23.765308+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.189:0/2620105210 expires 2024-09-14T14:17:29.088581+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1221135323 expires 2024-09-14T14:07:45.879701+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/580144606 expires 2024-09-14T14:07:45.879701+0000 2024-09-13T14:25:12.582 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1928248055 expires 2024-09-14T14:07:23.765308+0000 2024-09-13T14:25:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:12 smithi189 ceph-mon[90992]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:13.292 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 86 up_thru 101 down_at 83 last_clean_interval [8,82) [v2:172.21.15.17:6802/59727078,v1:172.21.15.17:6803/59727078] [v2:172.21.15.17:6804/59727078,v1:172.21.15.17:6805/59727078] exists,up b9a98879-a576-4dbf-80ce-247ecc208028', 'osd.1 up in weight 1 up_from 91 up_thru 101 down_at 88 last_clean_interval [13,87) [v2:172.21.15.17:6810/3641293563,v1:172.21.15.17:6811/3641293563] [v2:172.21.15.17:6812/3641293563,v1:172.21.15.17:6813/3641293563] exists,up f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2', 'osd.2 up in weight 1 up_from 96 up_thru 101 down_at 93 last_clean_interval [18,92) [v2:172.21.15.17:6818/847801397,v1:172.21.15.17:6819/847801397] [v2:172.21.15.17:6820/847801397,v1:172.21.15.17:6821/847801397] exists,up eb11f41d-35bc-4180-a0f6-781c7dbe5b5b', 'osd.3 up in weight 1 up_from 101 up_thru 101 down_at 98 last_clean_interval [26,97) [v2:172.21.15.17:6826/594040290,v1:172.21.15.17:6827/594040290] [v2:172.21.15.17:6828/594040290,v1:172.21.15.17:6829/594040290] exists,up 76cf0d9c-0405-4601-9c2c-3cbbba55dab8', 'osd.4 up in weight 1 up_from 31 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6800/3136403996,v1:172.21.15.189:6801/3136403996] [v2:172.21.15.189:6802/3136403996,v1:172.21.15.189:6803/3136403996] exists,up 698eddb3-30d9-431e-9731-9c37441f4359', 'osd.5 up in weight 1 up_from 37 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6808/379852919,v1:172.21.15.189:6809/379852919] [v2:172.21.15.189:6810/379852919,v1:172.21.15.189:6811/379852919] exists,up 60f030c1-c222-4a1e-b7d6-d788f37bc5eb', 'osd.6 up in weight 1 up_from 43 up_thru 98 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6816/2742951256,v1:172.21.15.189:6817/2742951256] [v2:172.21.15.189:6818/2742951256,v1:172.21.15.189:6819/2742951256] exists,up a53bc1f8-35e0-4e97-a9d1-90b318be2165', 'osd.7 up in weight 1 up_from 49 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.189:6824/1790039107,v1:172.21.15.189:6825/1790039107] [v2:172.21.15.189:6826/1790039107,v1:172.21.15.189:6827/1790039107] exists,up a3fdb8ae-2c63-493d-b2ac-8009b024fa12'] 2024-09-13T14:25:13.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-13T14:25:13.549 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:13.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:13 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/512814507' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:25:13.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:13 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/512814507' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:25:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:13 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/512814507' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:25:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:14 smithi017 ceph-mon[107556]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:14.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:14 smithi017 ceph-mon[103001]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:14 smithi189 ceph-mon[90992]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:15.295 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:25:15.295 INFO:teuthology.orchestra.run.smithi017.stdout: "name": "mon.a", 2024-09-13T14:25:15.295 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster": "ceph", 2024-09-13T14:25:15.295 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-13T14:25:15.295 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket_mode": "", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_client_required": "cephx, none", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_cluster_required": "cephx", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_debug": "true", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_required": "cephx", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_supported": "", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio": "true", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-13T14:25:15.296 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_poll_ms": "250", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_reap_max": "16", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_async_discard_threads": "0", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_block_size": "4096", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio": "false", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_inflight_ios": "false", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_enable_discard": "false", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry": "3", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash": "0", 2024-09-13T14:25:15.297 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring": "false", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_hipri": "false", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_type": "", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_alloc_size": "1048576", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_allocator": "hybrid", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_buffered_io": "true", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_for_zeros": "false", 2024-09-13T14:25:15.298 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_compact_log_sync": "false", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_log_runway": "4194304", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_prefetch": "1048576", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_flush_size": "524288", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_log_runway": "1048576", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery": "false", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-13T14:25:15.299 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_sync_write": "false", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocation_from_file": "true", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocator": "hybrid", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_async_db_compaction": "true", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_multi": "true", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-13T14:25:15.300 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_create": "true", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_create": "false", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_path": "", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_size": "0", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_path": "", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_preallocate_file": "false", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_size": "107374182400", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_create": "false", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_path": "", 2024-09-13T14:25:15.301 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_size": "100663296", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs": "true", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune": "true", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-13T14:25:15.302 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size": "0", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_type": "2q", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_clone_cow": "true", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-13T14:25:15.303 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_mode": "none", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_csum_type": "crc32c", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_freelist": "false", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-13T14:25:15.304 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_small_allocations": "0", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_read": "true", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_write": "false", 2024-09-13T14:25:15.305 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fail_eio": "false", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-13T14:25:15.306 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-13T14:25:15.307 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_ignore_data_csum": "false", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_op_age": "5.000000", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_alloc_size": "0", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size": "0", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_deferred_txc": "32", 2024-09-13T14:25:15.308 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size": "0", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_nid_prealloc": "1024", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_retry_disk_reads": "3", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cf": "true", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-13T14:25:15.309 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_mem": "512", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-13T14:25:15.310 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_tracing": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_write_v2": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_write_v2_random": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_zero_block_detection": "false", 2024-09-13T14:25:15.311 INFO:teuthology.orchestra.run.smithi017.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-13T14:25:15.312 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_signatures": "false", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_version": "2", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_signatures": "false", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_version": "2", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_signatures": "false", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_version": "2", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_sign_messages": "true", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "chdir": "", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_acl_type": "", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_asio_thread_count": "2", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_mid": "0.750000", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_size": "16384", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_caps_release_delay": "5", 2024-09-13T14:25:15.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_check_pool_perm": "true", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_force_sync_read": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_getattr_caps": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_inject_features": "", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_inject_tick_delay": "0", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_remount": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_dirsize_rbytes": "true", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_force_lazyio": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_fs": "", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_release_failure": "false", 2024-09-13T14:25:15.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_inline_size": "4096", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mds_namespace": "", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_metadata": "", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_gid": "-1", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_timeout": "300", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_uid": "-1", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mountpoint": "/", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_notify_timeout": "10", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc": "true", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty": "104857600", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_objects": "1000", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_size": "209715200", 2024-09-13T14:25:15.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_target_dirty": "8388608", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_permissions": "true", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota": "true", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota_df": "true", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_bytes": "0", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_periods": "4", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_min": "131072", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_reconnect_stale": "false", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_shutdown_timeout": "30", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_snapdir": ".snap", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_tick_interval": "1", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_trace": "", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_try_dentry_invalidate": "false", 2024-09-13T14:25:15.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_faked_inos": "false", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_random_mds": "false", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog": "false", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_port": "12201", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_monitors": "default=true", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog": "false", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_level": "info", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_addr": "-", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network": "", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network_interface": "", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_isal": "false", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_level": "5", 2024-09-13T14:25:15.317 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_winsize": "-15", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zstd_level": "1", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_op_num_threads": "6", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_stat_interval": "0", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_seastar_cpu_cores": "", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_seastar_num_threads": "0", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location": "", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook": "", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook_timeout": "10", 2024-09-13T14:25:15.318 INFO:teuthology.orchestra.run.smithi017.stdout: "daemonize": "false", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "daos_pool": "tank", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_alienstore": "0/5", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_allow_any_pool_priority": "false", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok": "1/5", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok_assert_abort": "false", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asserts_on_shutdown": "true", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_auth": "1/5", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bdev": "1/3", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluefs": "1/5", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluestore": "1/5", 2024-09-13T14:25:15.319 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_buffer": "0/1", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ceph_dedup": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ceph_exporter": "1/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephfs_mirror": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephsqlite": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_client": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_compressor": "1/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_context": "0/1", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crimson_interrupt": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crush": "1/1", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crypto": "1/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cyanstore": "0/5", 2024-09-13T14:25:15.320 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_deliberately_leak_memory": "false", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_disable_randomized_ping": "false", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_dpdk": "1/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_eventtrace": "1/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filer": "0/1", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filestore": "1/3", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_finisher": "1/1", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_fuse": "1/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeat_testing_span": "0", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeatmap": "1/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_javaclient": "1/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journal": "1/3", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journaler": "0/5", 2024-09-13T14:25:15.321 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_kstore": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_lockdep": "0/1", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mclock": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_balancer": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_locker": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log_expire": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_migrator": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_quiesce": "3/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_memstore": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgr": "2/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgrc": "1/5", 2024-09-13T14:25:15.322 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mon": "20/20", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_monc": "0/10", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ms": "1/1", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_none": "0/5", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objclass": "0/5", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objectcacher": "0/5", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objecter": "0/1", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_optracker": "0/5", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_osd": "1/5", 2024-09-13T14:25:15.323 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_paxos": "20/20", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_perfcounter": "1/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_prioritycache": "1/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rados": "0/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd": "0/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_mirror": "0/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_pwl": "0/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_replay": "0/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_refs": "0/0", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_reserver": "1/1", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw": "1/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_access": "1/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_datacache": "1/5", 2024-09-13T14:25:15.324 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_dbstore": "1/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_flight": "1/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_notification": "1/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_sync": "1/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rocksdb": "4/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_backref": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cache": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cleaner": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_device": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_epm": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-13T14:25:15.325 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_journal": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_lba": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_odata": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_omap": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_onode": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_t": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_tm": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_striper": "0/1", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_test": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_throttle": "1/1", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_timer": "0/1", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_tp": "0/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_trace": "1/5", 2024-09-13T14:25:15.326 INFO:teuthology.orchestra.run.smithi017.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "device_failure_prediction_mode": "none", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_graylog": "false", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_journald": "false", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_stderr": "true", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_syslog": "false", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event_tracing": "false", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_addr": "0.0.0.0", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_cert_file": "", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_http_port": "9926", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_key_file": "", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_prio_limit": "5", 2024-09-13T14:25:15.327 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sort_metrics": "true", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_stats_period": "5", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "fake_statfs_for_testing": "0", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "fatal_signal_handlers": "true", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_purge_ops": "10", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_truncate_ops": "128", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_apply_finisher_threads": "1", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_blackhole": "false", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_clone_range": "true", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_snap": "true", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_caller_concurrency": "10", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_collect_device_partition_information": "true", 2024-09-13T14:25:15.328 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_commit_timeout": "600.000000", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_inject_read_err": "false", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_omap_check": "false", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_verify_split": "false", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_dump_file": "", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fadvise": "true", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fail_eio": "true", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_shards": "16", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_size": "128", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap": "false", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap_threshold": "4096", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-13T14:25:15.329 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_inject_stall": "0", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_parallel": "false", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_trailing": "false", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_writeahead": "false", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_kill_at": "0", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs": "0", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-13T14:25:15.330 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size": "0", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_merge_threshold": "-10", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_odsync_write": "false", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend": "rocksdb", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend_path": "", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-13T14:25:15.331 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_timeout": "60", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_threads": "2", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_punch_hole": "false", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-13T14:25:15.332 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_ops": "50", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_seek_data_hole": "false", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc": "false", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_splice": "false", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_multiple": "2", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_rand_factor": "20", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_update_to": "1000", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-13T14:25:15.333 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_enable": "true", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-13T14:25:15.334 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_xfs_extsize": "false", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_zfs_snap": "false", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fio_dir": "/tmp/fio", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fsid": "523f52e4-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_allow_other": "true", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_atomic_o_trunc": "true", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_big_writes": "true", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_debug": "false", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_default_permissions": "false", 2024-09-13T14:25:15.335 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_disable_pagecache": "false", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_max_write": "0", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_multithreaded": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_require_active_mds": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_set_user_groups": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_move": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_read": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_write": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_use_invalidate_cb": "true", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_target_name": "ceph", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_file": "", 2024-09-13T14:25:15.336 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_inject_failure": "0", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_interval": "5", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "host": "smithi017", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-13T14:25:15.337 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "inject_early_sigterm": "false", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_agent_port": "6799", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_tracing_enable": "false", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_aio": "true", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_align_min_size": "65536", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_align": "true", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_size": "4096", 2024-09-13T14:25:15.338 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_dio": "true", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_discard": "false", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_force_aio": "false", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_ignore_corruption": "false", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_bytes": "10485760", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_entries": "100", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_replay_from": "0", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_write_header_frequency": "0", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_zero_on_create": "false", 2024-09-13T14:25:15.339 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prefetch_periods": "10", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prezero_periods": "5", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_write_head_interval": "15", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "key": "", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "keyfile": "", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_backend": "rocksdb", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_default_stripe_size": "65536", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount": "false", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_bytes": "67108864", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_ops": "512", 2024-09-13T14:25:15.340 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_nid_prealloc": "1024", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_onode_map_size": "1024", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_submit_transaction": "false", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_transaction": "false", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "librados_thread_count": "2", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep": "false", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep_force_backtrace": "false", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_coarse_timestamps": "true", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_flush_on_exit": "false", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_host": "127.0.0.1", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_port": "12201", 2024-09-13T14:25:15.341 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_new": "1000", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_recent": "10000", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stderr_prefix": "", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stop_at_utilization": "0.970000", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_file": "true", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_graylog": "false", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_journald": "false", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_stderr": "false", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_syslog": "false", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "max_rotating_auth_attempts": "10", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_action_on_write_error": "1", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_alternate_name_max": "8192", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_asio_thread_count": "2", 2024-09-13T14:25:15.342 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_export_pin": "true", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_dirs": "true", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_interval": "5", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_interval": "10", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max": "-1", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max_until": "-1", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_merge_size": "50", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_midchunk": "0.300000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_start": "0.200000", 2024-09-13T14:25:15.343 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_minchunk": "0.001000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_mode": "0", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_max": "1.200000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_min": "0.800000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_overload_epochs": "2", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_bits": "3", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_size": "10000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_target_decay": "10.000000", 2024-09-13T14:25:15.344 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_grace": "15.000000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_interval": "4.000000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_mid": "0.700000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_delay": "0", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_release_free_interval": "10", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_reservation": "0.050000", 2024-09-13T14:25:15.345 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_interval": "1", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_threshold": "262144", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_prealloc_inos": "1000", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_connect_bootstrapping": "false", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_damage_table_max_entries": "10000", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_auth_pins": "false", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_frag": "false", 2024-09-13T14:25:15.346 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_scatterstat": "false", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_subtrees": "false", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_decay_halflife": "5.000000", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_default_dir_hash": "2", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_defer_session_stale": "true", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_deny_all_reconnect": "false", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_keys_per_op": "16384", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_commit_size": "10", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_entries": "0", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_prefetch": "true", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_on_map": "false", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-13T14:25:15.347 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_early_reply": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enable_op_tracker": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enforce_unique_name": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-13T14:25:15.348 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_summarize_threshold": "10", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_health_dummy": "false", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_migrator_session_race": "false", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_join_fs": "", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_journal_format": "1", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-13T14:25:15.349 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_create_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_dirfrag_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_export_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_import_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_expire_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_replay_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_link_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_mdstable_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_openc_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_rename_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_shutdown_at": "0", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_event_large_threshold": "524288", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_events_per_segment": "1024", 2024-09-13T14:25:15.350 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_events": "-1", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_segments": "128", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_pause": "false", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_segment_size": "0", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_threshold": "128", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_warn_factor": "2.000000", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_caps_per_client": "1048576", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_flushes": "100000", 2024-09-13T14:25:15.351 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_requests": "100000", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_export_size": "20971520", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_file_recover": "32", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_files": "64", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops": "8192", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_snaps_per_dir": "100", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_metrics_update_interval": "2", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_per_client": "100", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_working_set": "10000", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_numa_node": "-1", 2024-09-13T14:25:15.352 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_complaint_time": "30.000000", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_duration": "600", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_size": "20", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_slow_op_size": "20", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_log_threshold": "5", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_grace": "15", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_interval": "5", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_caps": "30000", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-13T14:25:15.353 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_threshold": "262144", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_interval": "1.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_gid": "0", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_uid": "0", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-13T14:25:15.354 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_shutdown_check": "0", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_skip_ino": "0", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_max_uid": "4294967294", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_min_uid": "0", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_rstat": "false", 2024-09-13T14:25:15.355 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_standby_replay_damaged": "false", 2024-09-13T14:25:15.356 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_symlink_recovery": "true", 2024-09-13T14:25:15.357 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-13T14:25:15.357 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_exports": "0", 2024-09-13T14:25:15.357 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_fragments": "0", 2024-09-13T14:25:15.357 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_tick_interval": "5.000000", 2024-09-13T14:25:15.357 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_valgrind_exit": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_backtrace": "1", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_scatter": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_sessions": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mempool_debug": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_device_bytes": "1073741824", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_set": "false", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_size": "65536", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_bytes": "134217728", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_messages": "512", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-13T14:25:15.358 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_disabled_modules": "", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_enable_op_tracker": "true", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_max_pg_creating": "1024", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_max_pg_num_change": "128", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_bytes": "134217728", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_messages": "128", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-13T14:25:15.359 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_bytes": "134217728", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_messages": "128", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_duration": "600", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_size": "20", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_log_threshold": "5", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_bytes": "536870912", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_messages": "8192", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_pool": "true", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_standby_modules": "true", 2024-09-13T14:25:15.360 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_period": "5", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_threshold": "5", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_tick_period": "2", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_delete": "true", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_size_one": "false", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_auth_validate_all_caps": "true", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_bytes": "104857600", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_directed_command_retry": "2", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-13T14:25:15.361 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_parallel": "3", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_log_interval": "1.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_interval": "10.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_target_rank": "-1", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_level": "debug", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_file": "true", 2024-09-13T14:25:15.362 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_journald": "true", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_bootstrap": "false", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_start": "false", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_trim": "true", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-13T14:25:15.363 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cpu_threads": "4", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_crush_min_required_version": "hammer", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_daemon_bytes": "419430400", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_crit": "5", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_warn": "5", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_size_warn": "16106127360", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_json": "false", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_transactions": "false", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_extra_checks": "false", 2024-09-13T14:25:15.364 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_reef": "false", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_squid": "false", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_down_mkfs_grace": "60", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_down_uptime_grace": "60", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_default_strategy": "1", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_timeout": "5.000000", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_divisor": "2", 2024-09-13T14:25:15.365 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_enable_op_tracker": "true", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_fake_pool_delete": "false", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_force_quorum_join": "false", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_globalid_prealloc": "10000", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_detail_to_clog": "true", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_log_update_period": "5", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_max_detail": "50", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog": "true", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_interval": "600", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host": "[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host_override": "", 2024-09-13T14:25:15.366 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_initial_members": "", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease": "5.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_full_interval": "50", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max": "10000", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max_summary": "50", 2024-09-13T14:25:15.367 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_epochs": "500", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_osd": "10000", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pg_per_osd": "10000", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pool_pg_num": "65536", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_force_trim_to": "0", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_skip_sanity": "true", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_autotune": "true", 2024-09-13T14:25:15.368 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_target": "2147483648", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_beacon_grace": "30", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_digest_period": "5", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_inactive_grace": "60", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_min_osdmap_epochs": "500", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_complaint_time": "30", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_duration": "600", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_size": "20", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_size": "20", 2024-09-13T14:25:15.369 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_log_threshold": "5", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_in": "false", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size": "500", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-13T14:25:15.370 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_interval": "600", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_force_trim_to": "0", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_down_reporters": "2", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-13T14:25:15.371 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_report_timeout": "900", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-13T14:25:15.372 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_stuck_threshold": "60", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_probe_timeout": "2.000000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_change": "0.050000", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_osds": "4", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-13T14:25:15.373 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_interval": "86400", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_max_keys": "100", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_timeout": "300", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_session_timeout": "300", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_smart_report_timeout": "5", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stat_smooth_intervals": "6", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_min_size": "2", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_size": "4", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-13T14:25:15.374 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_debug": "false", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_provider_kill_at": "0", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_requester_kill_at": "0", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_timeout": "60.000000", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_target_pg_per_osd": "100", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_tick_interval": "5", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_interval": "300.000000", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_use_min_delay_socket": "false", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_older_version_delay": "604800", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-13T14:25:15.375 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_misplaced": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_older_version": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-13T14:25:15.376 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "monmap": "", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_tracing_enabled": "false", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_op_threads": "3", 2024-09-13T14:25:15.377 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_cm": "false", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_device_name": "", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_dscp": "96", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_local_gid": "", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_port_num": "1", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_sl": "3", 2024-09-13T14:25:15.378 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_support_srq": "true", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_type": "ib", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_reap_threshold": "5", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_before_connect": "false", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv4": "true", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv6": "false", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr1": "true", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr2": "true", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_max": "7568", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_min": "6800", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_count": "3", 2024-09-13T14:25:15.379 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_delay": "5", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_client": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mds": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mgr": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mon": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_osd": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_client_mode": "crc secure", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_mode": "crc secure", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_type": "", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_compress_secure": "false", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_idle_timeout": "900", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_ready_timeout": "10", 2024-09-13T14:25:15.380 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_data": "true", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_header": "true", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bad_msg": "false", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bug": "true", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_old_message": "true", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_skipped_message": "false", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_coremask": "0xF", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_enable_tso": "true", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-13T14:25:15.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hugepages": "", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_lro": "true", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_memory_channel": "4", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_pmd": "", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_port_id": "0", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_on_send": "false", 2024-09-13T14:25:15.382 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_initial_backoff": "0.200000", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_max": "1.000000", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_type": "", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_network_congestion": "0", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_socket_failures": "0", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_learn_addr_from_peer": "true", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_accept_failures": "4", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_backoff": "15.000000", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_client_mode": "secure crc", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_service_mode": "secure crc", 2024-09-13T14:25:15.383 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_min_size": "1024", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_mode": "none", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_min_cost": "65536", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_public_type": "", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_service_mode": "crc secure", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_listen_backlog": "512", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_nodelay": "true", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_rcvbuf": "0", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_type": "async+posix", 2024-09-13T14:25:15.384 INFO:teuthology.orchestra.run.smithi017.stdout: "no_config_file": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_completion_locks_per_session": "32", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_ops": "1024", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_tick_interval": "5.000000", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_timeout": "10.000000", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objectstore_blackhole": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-13T14:25:15.385 INFO:teuthology.orchestra.run.smithi017.stdout: "openssl_engine_opts": "", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_delay_time": "5.000000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_hist_halflife": "1000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_low_ops": "2", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_ops": "4", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_slop": "0.020000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_asio_thread_count": "2", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_async_recovery_min_cost": "100", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-13T14:25:15.386 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_max": "512", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_min": "64", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_degraded": "false", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_peering": "false", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_unfound": "true", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_beacon_report_interval": "300", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_duration": "30", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_max_block_size": "67108864", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blkin_trace_all": "false", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-13T14:25:15.387 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_for_log_corruption": "false", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_update_on_start": "true", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_cap": "256", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_size_cap": "524288000", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_op_priority": "63", 2024-09-13T14:25:15.388 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_watch_timeout": "30", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_command_max_records": "256", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_compact_on_start": "false", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_on_start": "true", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_weight_set": "true", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_feed_pullee": "-1", 2024-09-13T14:25:15.389 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_misdirected_ops": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_acting_change": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_purge_strays": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_op_order": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_shutdown": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-13T14:25:15.390 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_trim_objects": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_snaps": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_keys": "1024", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_stride": "524288", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-13T14:25:15.391 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_notify_timeout": "30", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep": "0.000000", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_discard_disconnected_ops": "true", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_ec_partial_reads": "true", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_enable_op_tracker": "true", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_info": "true", 2024-09-13T14:25:15.392 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown": "true", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_function_tracing": "false", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_grace": "20", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_interval": "6", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_peers": "10", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_size": "2000", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_stale": "600", 2024-09-13T14:25:15.393 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_max_size": "100000", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_min_size": "1000", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_size": "5120", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_kill_backfill_at": "0", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_cache_size": "50", 2024-09-13T14:25:15.394 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_dedup": "true", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max": "40", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_share_max_epochs": "40", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_name_len": "100", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_size": "0", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_backfills": "1", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_count": "5", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_period": "600", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_name_len": "2048", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_namespace_len": "256", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_size": "134217728", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-13T14:25:15.395 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_blocked_by": "16", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_log_entries": "10000", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pgls": "1024", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_cost": "8388608", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_objects": "10", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_scrubs": "3", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_trimming_pgs": "2", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_op_reply_len": "64", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_size": "90", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-13T14:25:15.396 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_profile": "balanced", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-13T14:25:15.397 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_base": "805306368", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_min": "134217728", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target": "4294967296", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_autotune": "false", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-13T14:25:15.398 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_pg_log_entries": "250", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_recovery_priority": "0", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_interval": "5", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_cache_shards": "32", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_op_tracker_shard": "32", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_auto_affinity": "true", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_node": "-1", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_prefer_iface": "true", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objecter_finishers": "1", 2024-09-13T14:25:15.399 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore": "bluestore", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_fuse": "false", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_tracing": "false", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_complaint_time": "30.000000", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_duration": "600", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_size": "20", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_size": "20", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_log_threshold": "5", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards": "0", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_hdd": "5", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_ssd": "8", 2024-09-13T14:25:15.400 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_min_cost": "65536", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue_cut_off": "high", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_timeout": "15", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_open_classes_on_start": "true", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_os_flags": "0", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_peering_op_priority": "255", 2024-09-13T14:25:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_cost": "1048576", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_priority": "5", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_max": "10000", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_min": "100", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-13T14:25:15.402 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crimson": "false", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-13T14:25:15.403 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flags": "0", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_min_size": "0", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_num": "32", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pgp_num": "0", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_read_ratio": "70", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_size": "3", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_type": "replicated", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-13T14:25:15.404 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_push_per_object_cost": "1000", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap": "true", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_cost": "20971520", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active": "0", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_single_start": "1", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_priority": "3", 2024-09-13T14:25:15.405 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_priority": "5", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep": "0.000000", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_repair_during_recovery": "false", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_requested_scrub_priority": "120", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair": "false", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-13T14:25:15.406 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_hour": "0", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_week_day": "0", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_max": "25", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_min": "5", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_cost": "52428800", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_during_recovery": "false", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_hour": "0", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_week_day": "0", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_event_cost": "4096", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_invalid_stats": "true", 2024-09-13T14:25:15.407 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_preemptions": "5", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_priority": "5", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_sleep": "0.000000", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_skip_data_digest": "false", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_smart_report_timeout": "5", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_cost": "1048576", 2024-09-13T14:25:15.408 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_priority": "5", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_transaction_size": "30", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-13T14:25:15.409 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tracing": "false", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_use_stale_snap": "false", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "osdc_blkin_trace_all": "false", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_kill_at": "0", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_max_join_drift": "10", 2024-09-13T14:25:15.410 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min": "500", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min_wait": "0.050000", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_propose_interval": "1.000000", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max": "500", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_min": "250", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_stash_full_interval": "25", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_max": "500", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_min": "250", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "perf": "true", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "pid_file": "", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-13T14:25:15.411 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addr": "-", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addrv": "", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "public_bind_addr": "-", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network": "172.21.0.0/20", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network_interface": "", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_busy_polling": "false", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_enabled": "false", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_session_max_number": "256", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_mon_op_timeout": "0", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_osd_op_timeout": "0", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_tracing": "false", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_atime_update_interval": "60", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-13T14:25:15.412 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_parent_reads": "false", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_snap_reads": "false", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blkin_trace_all": "false", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache": "true", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_policy": "writearound", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_size": "33554432", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-13T14:25:15.413 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_clone_copy_on_read": "false", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_compression_hint": "none", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_concurrent_management_ops": "10", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_clone_format": "auto", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_data_pool": "", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_features": "61", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_format": "2", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_map_options": "", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_order": "22", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_pool": "rbd", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_count": "0", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_unit": "0", 2024-09-13T14:25:15.414 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_enable_alloc_hint": "true", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler": "simple", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-13T14:25:15.415 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_order": "24", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_pool": "", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_splay_width": "4", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_parent_reads": "false", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_snap_reads": "false", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-13T14:25:15.416 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-13T14:25:15.417 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-13T14:25:15.418 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mtime_update_interval": "60", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_non_blocking_aio": "true", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_thread_timeout": "60", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_threads": "1", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_parent_cache_enabled": "false", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_plugins": "", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst": "0", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-13T14:25:15.419 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_limit": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_exclude_ops": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_limit": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-13T14:25:15.420 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_read_from_replica_policy": "default", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_skip_partial_discard": "true", 2024-09-13T14:25:15.421 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_tracing": "false", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_names": "true", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_pool": "true", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_base_url": "", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_log_level": "", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_acl_grants_max_num": "100", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_admin_entry": "admin", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_asio_assert_yielding": "false", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_backend_store": "rados", 2024-09-13T14:25:15.422 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_barbican_url": "", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_beast_enable_async": "true", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_counters_cache": "false", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_enabled": "true", 2024-09-13T14:25:15.423 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_expiry_interval": "900", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_lru_size": "25000", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_config_store": "rados", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_content_length_compat": "false", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress": "true", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cors_rules_max_num": "100", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cross_domain_policy": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_addr": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-13T14:25:15.424 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_password": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_username": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_require_ssl": "true", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-13T14:25:15.425 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_addr": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_auth": "token", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_namespace": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_prefix": "", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-13T14:25:15.426 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_token_file": "", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_buffersize": "524288", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_time": "300", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-13T14:25:15.427 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-13T14:25:15.428 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_changes_size": "1000", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_num_shards": "128", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_window": "30", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-13T14:25:15.429 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_disable_s3select": "false", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-13T14:25:15.430 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_name": "", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_s3website_name": "", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding": "true", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-13T14:25:15.431 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_gc_threads": "true", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_lc_threads": "true", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_mdsearch": "true", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_ops_log": "false", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_quota_threads": "true", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_static_website": "false", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_usage_log": "false", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enforce_swift_acls": "true", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_exit_timeout_secs": "120", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_expose_bucket": "false", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_extended_http_attrs": "", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_filter": "none", 2024-09-13T14:25:15.432 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontends": "beast port=7480", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred": "50", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_objs": "32", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_period": "3600", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-13T14:25:15.433 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_init_timeout": "300", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_connection_idle": "300", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-13T14:25:15.434 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_domain": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password_path": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_project": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_tenant": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token_path": "", 2024-09-13T14:25:15.435 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_user": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_domain": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_password": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_project": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_user": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_url": "", 2024-09-13T14:25:15.436 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_debug_interval": "-1", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_lock_max_time": "90", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_objs": "32", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_rules": "1000", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_worker": "3", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_thread_delay": "0", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_dnattr": "uid", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchfilter": "", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-13T14:25:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_uri": "", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_http_headers": "", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name_utc": "false", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-13T14:25:15.438 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_name_len": "0", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_size": "0", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_chunk_size": "4194304", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_listing_results": "1000", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_notify_retries": "10", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_param_size": "1048576", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_size": "5368709120", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_slo_entries": "1000", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_log_max_shards": "64", 2024-09-13T14:25:15.439 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mp_lock_max_time": "600", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-13T14:25:15.440 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_max_gc": "300", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_async_rados_threads": "32", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_control_oids": "8", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_numa_node": "-1", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_chunk_size": "100", 2024-09-13T14:25:15.441 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_gc_interval": "600", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_timeout": "600", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_tracing": "false", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_token": "", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_url": "", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_verify_ssl": "true", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_rados": "false", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_socket_path": "", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-13T14:25:15.442 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval": "2.000000", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_lanes": "3", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_partitions": "3", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_continue": "true", 2024-09-13T14:25:15.443 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_tracing": "false", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm": "", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_id": "", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region": "", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-13T14:25:15.444 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_request_uri": "", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_batch_size": "64", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_max_aio": "128", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_num_logs": "16", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_thread_interval": "600", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_resolve_cname": "false", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_run_sync_thread": "true", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-13T14:25:15.445 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_scheduler_type": "throttler", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_script_uri": "", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_service_provider_name": "", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_id": "", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_secret": "", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_entry": "sts", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_key": "sts", 2024-09-13T14:25:15.446 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_min_session_duration": "900", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_token_introspection_url": "", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_account_in_url": "false", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_entry": "auth", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_url": "", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_custom_header": "", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_need_stats": "true", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_tenant_name": "", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_token_expiration": "86400", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url": "", 2024-09-13T14:25:15.447 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url_prefix": "swift", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_lease_period": "120", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-13T14:25:15.448 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_thread_pool_size": "512", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_comment": "", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_createby": "", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_encoding": "", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_flag": "false", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_origin": "", 2024-09-13T14:25:15.449 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_tracker": "", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_trust_forwarded_https": "false", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_shards": "32", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_user_shards": "1", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_use_opa_authz": "false", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_counters_cache": "false", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_max_buckets": "1000", 2024-09-13T14:25:15.450 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_policies_max_num": "100", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_unique_email": "true", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_verify_ssl": "true", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone": "", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_id": "", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup": "", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_id": "", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-13T14:25:15.451 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_auto_session_clear": "true", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_block_size": "4096", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_size": "536870912", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-13T14:25:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_index_type": "binary_search", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_partition_filters": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_perf": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "run_dir": "/var/run/ceph", 2024-09-13T14:25:15.453 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_block_create": "true", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cache_lru_size": "67108864", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_max_object_size": "16777216", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_device_size": "53687091200", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_full_integrity_check": "false", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_capacity": "16", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_main_device_type": "SSD", 2024-09-13T14:25:15.454 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_segment_size": "67108864", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "set_keepcaps": "false", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "setgroup": "ceph", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser": "ceph", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser_match_path": "", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "thp": "false", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_default_timeout": "60", 2024-09-13T14:25:15.455 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout: "throttler_perf_counter": "true", 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout: "tmp_dir": "/tmp", 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout: "uadk_compressor_enabled": "false", 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-13T14:25:15.456 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:25:16.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-13T14:25:16.596 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:16.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:16 smithi017 ceph-mon[103001]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:16.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:16 smithi017 ceph-mon[107556]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:16 smithi189 ceph-mon[90992]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:17] "GET /metrics HTTP/1.1" 200 34096 "" "Prometheus/2.51.0" 2024-09-13T14:25:18.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:18.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:18.253 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:25:18.322 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2024-09-13T14:25:18.322 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:25:18.332 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2024-09-13T14:25:18.332 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:25:18.341 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2024-09-13T14:25:18.341 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:25:18.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:18 smithi017 ceph-mon[107556]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:18.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1050167960' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:25:18.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:18 smithi017 ceph-mon[103001]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:18.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1050167960' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:25:19.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:18 smithi189 ceph-mon[90992]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:19.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1050167960' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:25:19.266 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-13T14:25:19.266 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-13T14:25:19.267 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:25:19.268 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:25:19.269 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-13T14:25:19.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-13T14:25:19.269 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-13T14:25:19.269 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:25:19.270 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:25:19.270 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-13T14:25:19.270 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:25:19.270 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:25:19.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-13T14:25:19.270 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-13T14:25:19.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:19.271 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-13T14:25:19.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:25:19.271 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-13T14:25:19.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:25:19.272 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-13T14:25:19.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:25:19.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:19.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:19.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:19.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:19.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:19.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:19.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:19.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:19.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:19.789 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:19.801 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:19.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:19.815 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:19.827 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:19.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:19.931 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:19.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:20.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:20.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:20.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:20.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:20.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:20.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:20.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.431 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:20.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:20.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:20.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:20.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:20.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:20.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:20.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:20.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:20.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:20.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:20 smithi189 ceph-mon[90992]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:21.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:21.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:21.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:21.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:21.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:21.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:20 smithi017 ceph-mon[103001]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:21.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:20 smithi017 ceph-mon[107556]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:21.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:21.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:21.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:21.425 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:21.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:21.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:21.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:21.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:21.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:21.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:21.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:21.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:21.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:21.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:22.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:22.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:22.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:22.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:22.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:22.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:22.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:22.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:22.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:22.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:22.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:22.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:22.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:22.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:22.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:22.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:22.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:22.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:22.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:23.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:23.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:23.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:23 smithi017 ceph-mon[103001]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:23 smithi017 ceph-mon[107556]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:23.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:23.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:23.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:23.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:23.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:23 smithi189 ceph-mon[90992]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:23.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:23.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:23.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:23.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:23.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:23.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:23.941 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:23.968 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:23.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:23.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:24.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:24.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:24.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:24.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:24.351 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:24.453 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:24.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:24.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:24.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:24.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:24.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:24.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:24.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:24.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:24.990 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:24.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:25.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:25.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:25.156 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:25.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:25.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:25.308 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:25 smithi017 ceph-mon[107556]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:25.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:25 smithi017 ceph-mon[103001]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:25.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:25.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:25.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:25.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:25 smithi189 ceph-mon[90992]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:25.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:25.611 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:25:25.611 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:25:25.611 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:25:25.619 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:25.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:25.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:25.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:25.874 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:25.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:25.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:25.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:25.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:26.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:26.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:26.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:26.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:26.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:26.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:26.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:26.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:26.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:26.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:26.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:26.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:26.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:26.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:26.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:26.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:26.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:27.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:27.042 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:25:27.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:27.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:27.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:27 smithi017 ceph-mon[103001]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:27.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:27 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:25:27.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:27 smithi017 ceph-mon[107556]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:27.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:27 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:25:27.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:27.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:27.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:27.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:27.390 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:27] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.51.0" 2024-09-13T14:25:27.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:27 smithi189 ceph-mon[90992]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:27 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:25:27.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:27.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:27.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:27.735 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:27.761 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:27.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:27.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:27.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:27.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:27.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:27.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:28.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:28.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:28.092 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' created 2024-09-13T14:25:28.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:28.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:28.195 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:25:28.195 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:25:28.195 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:25:28.196 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:25:28.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:28.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:28.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:28.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:25:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:25:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:25:28.497 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/532704841' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:25:28.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:28.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:28.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:28.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:28.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:28.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:28.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:28.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:29.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:29.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:29.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:29.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:29.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:29.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:29.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[103001]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:29.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:25:29.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[103001]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:25:29.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[107556]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:29.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:25:29.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:29 smithi017 ceph-mon[107556]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:25:29.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:29.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:29.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:29.488 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:25:29.488 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:25:29.489 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:25:29.490 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:25:29.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:29 smithi189 ceph-mon[90992]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3153081416' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:25:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:29 smithi189 ceph-mon[90992]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:25:29.507 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:25:29.508 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:25:29.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:25:29.524 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:25:29.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:29.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:29.540 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:25:29.540 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:25:29.557 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:25:29.557 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:25:29.573 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:25:29.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:25:29.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:25:29.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:25:29.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:29.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:29.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:25:29.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:29.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:29.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:29.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:29.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:29.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:25:30.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:30.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:30.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:30 smithi017 ceph-mon[103001]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:25:30.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/377938129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:30.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:30 smithi017 ceph-mon[107556]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:25:30.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/377938129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:30.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:30.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:30.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:30.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:30.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:30.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:30.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:30.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:30 smithi189 ceph-mon[90992]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:25:30.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/377938129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:30.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:30.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:30.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:30.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:30.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:30.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:30.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:30.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:30.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:30.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:30.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:30.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:31.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:31.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:31.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:31.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:31.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:31.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:31.380 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:31.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[107556]: pgmap v245: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:25:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[107556]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:25:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[103001]: pgmap v245: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:31.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:25:31.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:31 smithi017 ceph-mon[103001]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:25:31.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:31.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-13T14:25:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:31 smithi189 ceph-mon[90992]: pgmap v245: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:31 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:25:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:31 smithi189 ceph-mon[90992]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:25:31.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:31.576 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:31.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:31.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:31.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:31.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:31.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:31.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:31.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:31.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:32.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:32.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:32.139 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:32.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:32.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:32.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.398 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:32.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:32.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:32.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:32.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:32.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:32.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:32.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:32.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:32.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:33.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:33.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:33.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:33.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:33.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:25:33.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:33.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:33 smithi017 ceph-mon[103001]: pgmap v247: 89 pgs: 1 creating+peering, 10 unknown, 78 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:33.251 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:33 smithi017 ceph-mon[107556]: pgmap v247: 89 pgs: 1 creating+peering, 10 unknown, 78 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:33.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:33.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:33.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:33.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:33 smithi189 ceph-mon[90992]: pgmap v247: 89 pgs: 1 creating+peering, 10 unknown, 78 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:33.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:33.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:33.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:33.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:33.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:33.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:33.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:33.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:33.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:33.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:34.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:34.017 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:34.029 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:25:34.029 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:25:34.030 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-13T14:25:34.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-13T14:25:34.038 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:34.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:34.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:34.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:34.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:34.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:34.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:34.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:34.543 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:34.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:34.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:34.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:34.796 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:34.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:34.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:34.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:34.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:35.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:35.048 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:35.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:35.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:35.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:35.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:35.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:35.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:35.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:35.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:35.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:35 smithi017 ceph-mon[103001]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:35.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:35 smithi017 ceph-mon[107556]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:35 smithi189 ceph-mon[90992]: pgmap v248: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:35.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:35.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:35.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:35.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:35.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:35.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:35.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:35.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:35.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:36.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:36.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:36.117 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:25:36.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:36.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:25:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:25:36.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:36.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:36.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:36.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:25:36.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:36.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:36.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:36.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:36.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:36.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:36.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:36.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:36.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:37.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:37.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:37.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:37.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:37.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:37.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:37.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:37.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:37.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:37.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:37.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: pgmap v249: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/857204115' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:25:37.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:25:37.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:25:37.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:25:37.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:37.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:37.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:37] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.51.0" 2024-09-13T14:25:37.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:37.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:37.679 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:37.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:37.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:37.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:37.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:37.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:37.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:25:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:25:38.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:38.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:38.307 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.330 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:38.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:38.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:38.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:38.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:38.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:25:38.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:38.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:38.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:38.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:38.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:38.952 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:38.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:38.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:39.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:39.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:39.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:39.124 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-13T14:25:39.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[103001]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1201068681' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[107556]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1201068681' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:39.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:39.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:39.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:25:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:39.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:39.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:39.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.473 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:39.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:39 smithi189 ceph-mon[90992]: pgmap v251: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1201068681' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:25:39.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:39.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:39.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:39.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:39.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:39.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:39.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:39.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:39.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:39.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:40.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:40.096 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:40.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:40.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:40.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:40.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:40.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:40.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:25:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:40 smithi017 ceph-mon[103001]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:25:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:25:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:40 smithi017 ceph-mon[107556]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:25:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:40.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:25:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:40 smithi189 ceph-mon[90992]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:25:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:40.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:40.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:40.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:40.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:40.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:40.822 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-13T14:25:40.824 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:25:40.824 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:25:40.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:40.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:40.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:40.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:25:40.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:25:40.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_147622 2024-09-13T14:25:40.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:25:40.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:25:40.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:41.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:41.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:41.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:41.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:41.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:41.307 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:41.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:41.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:41.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:41 smithi017 ceph-mon[107556]: pgmap v253: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:41.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:41 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:25:41.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:41 smithi017 ceph-mon[103001]: pgmap v253: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:41.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:41 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:25:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:41 smithi189 ceph-mon[90992]: pgmap v253: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:41 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:25:41.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:41.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:41.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:41.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:41.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:41.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:41.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:41.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3989 3973 248.302 248.312 0.00222366 0.00401236 2024-09-13T14:25:41.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:41.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:41.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:42.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:42.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:42.166 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:25:42.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:25:42.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:42.208 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:25:42.208 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:25:42.237 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:42.246 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:25:42.246 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:25:42.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:25:42.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:25:42.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2521017686' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:42.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2521017686' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:42.320 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:25:42.321 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:25:42.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:42.359 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:25:42.359 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:25:42.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:42.393 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:25:42.393 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:25:42.427 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:25:42.427 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:25:42.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2521017686' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:42.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:42.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:42.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:42.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:42.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:42.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7686 7670 239.672 231.062 0.00288492 0.00416473 2024-09-13T14:25:42.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:42.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:42.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:43.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:43.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:43.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:43.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:43.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:43.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:43.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:43.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:43 smithi017 ceph-mon[107556]: pgmap v254: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:43 smithi017 ceph-mon[103001]: pgmap v254: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:43.470 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:43 smithi189 ceph-mon[90992]: pgmap v254: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:25:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:25:43.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:43.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:43.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:43.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:43.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:43.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:43.873 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 11125 11109 231.42 214.938 0.00129671 0.00427527 2024-09-13T14:25:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:43.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:43.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:43.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:44.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:44.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:44.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:44.272 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:44.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:44.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:44.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:44.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:44.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:44.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:44.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:44.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:44.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:44.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:44.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:44.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:44.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 14445 14429 225.421 207.5 0.00304377 0.00442854 2024-09-13T14:25:44.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:45.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:45.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:45.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:45.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:25:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:45.261 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:45.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:45.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:45 smithi017 ceph-mon[103001]: pgmap v255: 89 pgs: 89 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s 2024-09-13T14:25:45.378 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:45 smithi017 ceph-mon[107556]: pgmap v255: 89 pgs: 89 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s 2024-09-13T14:25:45.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:45.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:45 smithi189 ceph-mon[90992]: pgmap v255: 89 pgs: 89 active+clean; 288 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s 2024-09-13T14:25:45.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:45.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:45.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:45.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:45.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:45.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:45.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 17439 17423 217.758 187.125 0.0034119 0.00458619 2024-09-13T14:25:45.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:45.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:45.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:46.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:46.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:46.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:46.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:46.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:46.406 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:46.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:46.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:46.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:46.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:46.631 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:46.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:46.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:46.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:46.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 20235 20219 210.588 174.75 0.00421943 0.00474309 2024-09-13T14:25:46.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:46.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:46.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:47.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:47.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:47.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:47.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:47.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:47.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:47 smithi017 ceph-mon[103001]: pgmap v256: 89 pgs: 89 active+clean; 609 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:25:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:47] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.51.0" 2024-09-13T14:25:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:47 smithi017 ceph-mon[107556]: pgmap v256: 89 pgs: 89 active+clean; 609 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:25:47.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:47.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:47.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:47 smithi189 ceph-mon[90992]: pgmap v256: 89 pgs: 89 active+clean; 609 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:25:47.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:47.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:47.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:47.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:47.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:47.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:47.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 23129 23113 206.341 180.875 0.0029804 0.00484056 2024-09-13T14:25:47.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:47.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:47.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:48.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:48.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:48.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:48.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:48.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:48.355 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:11119 5.7 e108/104 5:ed35bd60:::benchmark_data_smithi017_147622_object11118:head v 108'696, mlcod=108'693)", 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:43.847417+0000", 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4.4485685640000003, 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063536325000000005, 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.356 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847417+0000", 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847417+0000", 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847417+0000", 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7199999999999999e-07 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847602+0000", 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000185241 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847609+0000", 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7769999999999997e-06 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847616+0000", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1500000000000002e-06 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847668+0000", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1403000000000002e-05 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847681+0000", 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2876e-05 2024-09-13T14:25:48.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.910887+0000", 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063206415000000002 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.910953+0000", 2024-09-13T14:25:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5990999999999999e-05 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:11122 5.3 5:c2ea3b04:::benchmark_data_smithi017_147622_object11121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:43.847521+0000", 2024-09-13T14:25:48.361 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4.4484642609999998, 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.067289884999999994, 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 11122 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847521+0000", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847521+0000", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847522+0000", 2024-09-13T14:25:48.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7899999999999997e-07 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847539+0000", 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7469000000000002e-05 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847549+0000", 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5759999999999995e-06 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847554+0000", 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2719999999999997e-06 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847622+0000", 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7891000000000004e-05 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.847729+0000", 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001069 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.911538+0000", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063809421000000005 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.913323+0000", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017847679999999999 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.913362+0000", 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8933999999999997e-05 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.914722+0000", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001359797 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.914772+0000", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0185000000000001e-05 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:43.914811+0000", 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8792999999999999e-05 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.367 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:18228 5.3 5:cfd86637:::benchmark_data_smithi017_147622_object18227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.127176+0000", 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1688092079999999, 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.039760031000000001, 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.368 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18228 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127176+0000", 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127176+0000", 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127178+0000", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5859999999999999e-06 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127207+0000", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9383999999999999e-05 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127223+0000", 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5868000000000001e-05 2024-09-13T14:25:48.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127230+0000", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0450000000000003e-06 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127366+0000", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013573400000000001 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127491+0000", 2024-09-13T14:25:48.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125776 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127754+0000", 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026271299999999998 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166259+0000", 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038505222999999998 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166727+0000", 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000467794 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166740+0000", 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2711000000000001e-05 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166898+0000", 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015785500000000001 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166936+0000", 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8342e-05 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:18231 5.3 5:c5babefa:::benchmark_data_smithi017_147622_object18230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.127753+0000", 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.168232615, 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.039559432999999998, 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18231 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127753+0000", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127753+0000", 2024-09-13T14:25:48.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127754+0000", 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2299999999999999e-07 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127775+0000", 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1509999999999999e-05 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127784+0000", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1169999999999993e-06 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.127792+0000", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4279999999999999e-06 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131394+0000", 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003602212 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131520+0000", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126458 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131708+0000", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018765399999999999 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166968+0000", 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035260121999999998 2024-09-13T14:25:48.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167135+0000", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000167104 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167172+0000", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7321999999999999e-05 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167266+0000", 2024-09-13T14:25:48.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3522000000000003e-05 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167312+0000", 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6060999999999998e-05 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:18233 5.5 5:acde318d:::benchmark_data_smithi017_147622_object18232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.380 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.128056+0000", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1679295359999999, 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038253697000000003, 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18233 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128056+0000", 2024-09-13T14:25:48.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128056+0000", 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128057+0000", 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4e-07 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128076+0000", 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9672000000000001e-05 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128087+0000", 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1079e-05 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128094+0000", 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2269999999999996e-06 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128891+0000", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079700099999999998 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129069+0000", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177994 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129197+0000", 2024-09-13T14:25:48.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012788799999999999 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130986+0000", 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001788371 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131013+0000", 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6951999999999999e-05 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166206+0000", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035193651999999999 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166263+0000", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6619999999999997e-05 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166309+0000", 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6400999999999998e-05 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.386 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:18234 5.5 5:a22f93d5:::benchmark_data_smithi017_147622_object18233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.128113+0000", 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1678721259999998, 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038358102999999998, 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.387 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18234 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128113+0000", 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128113+0000", 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128114+0000", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9800000000000003e-07 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128133+0000", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.931e-05 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128141+0000", 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1829999999999994e-06 2024-09-13T14:25:48.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128147+0000", 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4269999999999999e-06 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129820+0000", 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016738809999999999 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129935+0000", 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114814 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130067+0000", 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013185399999999999 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131462+0000", 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013944059999999999 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166357+0000", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034895369000000002 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166398+0000", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1026999999999999e-05 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166440+0000", 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2166000000000003e-05 2024-09-13T14:25:48.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166471+0000", 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1068000000000001e-05 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:18230 5.b e108/104 5:d83fc576:::benchmark_data_smithi017_147622_object18229:head v 108'1180, mlcod=108'1178)", 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.128470+0000", 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1675152080000002, 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038656416999999998, 2024-09-13T14:25:48.393 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128470+0000", 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128470+0000", 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128471+0000", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7499999999999999e-07 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128492+0000", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1024999999999999e-05 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128503+0000", 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0886e-05 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.128509+0000", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3130000000000004e-06 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131168+0000", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026592429999999999 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131187+0000", 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8576999999999998e-05 2024-09-13T14:25:48.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167091+0000", 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035904351000000001 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167126+0000", 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5247000000000003e-05 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.397 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18228 5.3 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.129949+0000", 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1660360839999999, 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036317781, 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129949+0000", 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129949+0000", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129950+0000", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4600000000000004e-07 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129957+0000", 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2509999999999997e-06 2024-09-13T14:25:48.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129968+0000", 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0834e-05 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.129974+0000", 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2450000000000003e-06 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166244+0000", 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036269758999999999 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166252+0000", 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166267+0000", 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5238e-05 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.401 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18228 5.3 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.130008+0000", 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1659769930000001, 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036930337000000001, 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130008+0000", 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130008+0000", 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130009+0000", 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9100000000000004e-07 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130012+0000", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6899999999999998e-06 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130022+0000", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0270000000000001e-06 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130030+0000", 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7679999999999997e-06 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166714+0000", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036683398999999998 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166721+0000", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7419999999999997e-06 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166939+0000", 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021712000000000001 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.405 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:18232 5.b e108/104 5:df2b3038:::benchmark_data_smithi017_147622_object18231:head v 108'1181, mlcod=108'1178)", 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.130729+0000", 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1652563420000002, 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.040079969, 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130729+0000", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130729+0000", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130729+0000", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2699999999999999e-07 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130753+0000", 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3254000000000001e-05 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130763+0000", 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.91e-06 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.130771+0000", 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8319999999999995e-06 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166290+0000", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035518268999999998 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166312+0000", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2274000000000001e-05 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.170717+0000", 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044046379999999998 2024-09-13T14:25:48.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.170809+0000", 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2164999999999996e-05 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:18240 5.5 5:aeb10dc2:::benchmark_data_smithi017_147622_object18239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.131507+0000", 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1644779359999999, 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038204698000000002, 2024-09-13T14:25:48.410 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18240 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131507+0000", 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131507+0000", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131509+0000", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3850000000000001e-06 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131531+0000", 2024-09-13T14:25:48.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2095000000000001e-05 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131542+0000", 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1484999999999999e-05 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131549+0000", 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1590000000000004e-06 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131625+0000", 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5584e-05 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131741+0000", 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116193 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131853+0000", 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111635 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166505+0000", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034651948000000002 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166541+0000", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6443000000000003e-05 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.169352+0000", 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028108040000000001 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.169665+0000", 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031257000000000002 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.169712+0000", 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7397e-05 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.416 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18233 5.5 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.131860+0000", 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1641250150000002, 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034459970999999999, 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131860+0000", 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131860+0000", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131861+0000", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7899999999999999e-07 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131864+0000", 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1020000000000001e-06 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131870+0000", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7129999999999999e-06 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131882+0000", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1595e-05 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166191+0000", 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034309473 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166200+0000", 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0149999999999992e-06 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166320+0000", 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119694 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18234 5.5 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.420 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.131894+0000", 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1640913739999998, 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034469341000000001, 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131894+0000", 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131894+0000", 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131894+0000", 2024-09-13T14:25:48.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1199999999999999e-07 2024-09-13T14:25:48.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131905+0000", 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0890000000000001e-05 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131912+0000", 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6549999999999998e-06 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.131916+0000", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7210000000000002e-06 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166346+0000", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034429203999999998 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166352+0000", 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0290000000000002e-06 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166363+0000", 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.153e-05 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:18243 5.2 e108/104 5:4786c6ad:::benchmark_data_smithi017_147622_object18242:head v 108'1133, mlcod=108'1131)", 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.132904+0000", 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1630816830000001, 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034640437000000003, 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.426 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132904+0000", 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132904+0000", 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132904+0000", 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0999999999999998e-07 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132925+0000", 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0741e-05 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166167+0000", 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033242027 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166182+0000", 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4703e-05 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166297+0000", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115066 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166317+0000", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0496000000000002e-05 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167507+0000", 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001189884 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167544+0000", 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6810000000000002e-05 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18234 5.5 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.132999+0000", 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.162986123, 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033475931, 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.432 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132999+0000", 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.132999+0000", 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133000+0000", 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2e-07 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133002+0000", 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.74e-06 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166167+0000", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033164370999999998 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166182+0000", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5104e-05 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166387+0000", 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020530799999999999 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166392+0000", 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3149999999999999e-06 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166475+0000", 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2572999999999997e-05 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.437 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35034.0:18231 5.3 e108/104 ondisk, result = 0)", 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.133603+0000", 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1623823789999999, 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033533405000000002, 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133603+0000", 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133603+0000", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133603+0000", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5300000000000004e-07 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.133607+0000", 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0040000000000001e-06 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166166+0000", 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.032558337999999999 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166180+0000", 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.418e-05 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166958+0000", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077796699999999996 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.166964+0000", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6999999999999996e-06 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.167136+0000", 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017256300000000001 2024-09-13T14:25:48.441 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:19271 5.5 5:a6be887c:::benchmark_data_smithi017_147622_object19270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.509799+0000", 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.786185972, 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.05836881, 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:25:48.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 19271 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.509799+0000", 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.509799+0000", 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.509800+0000", 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1429999999999999e-06 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.509819+0000", 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8997000000000001e-05 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.563514+0000", 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.053694486999999999 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.563526+0000", 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2189e-05 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.564536+0000", 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010095 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.564672+0000", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013682899999999999 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.564805+0000", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013243799999999999 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.565914+0000", 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011089680000000001 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.567838+0000", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019243649999999999 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.568053+0000", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000214649 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.568119+0000", 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5740000000000004e-05 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.568168+0000", 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9505000000000001e-05 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:19279 5.e e108/104 5:7144387a:::benchmark_data_smithi017_147622_object19278:head v 108'1172, mlcod=108'1170)", 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.513544+0000", 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.7824411090000001, 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.054541113000000002, 2024-09-13T14:25:48.448 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513544+0000", 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513544+0000", 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513545+0000", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004e-06 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.566625+0000", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.053080377999999998 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.566643+0000", 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7028999999999999e-05 2024-09-13T14:25:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.566652+0000", 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3309999999999993e-06 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.566772+0000", 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120229 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.566792+0000", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0248e-05 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.568041+0000", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012486229999999999 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.568085+0000", 2024-09-13T14:25:48.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4270999999999997e-05 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:19280 5.c e108/104 5:3aed1e93:::benchmark_data_smithi017_147622_object19279:head v 108'1148, mlcod=108'1146)", 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:46.513578+0000", 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.7824069649999998, 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.053519098000000001, 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513578+0000", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513578+0000", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513579+0000", 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7100000000000002e-07 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.513600+0000", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1109000000000001e-05 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.565027+0000", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051426855 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.565038+0000", 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0951000000000001e-05 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.565100+0000", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1904999999999998e-05 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.565121+0000", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1841999999999999e-05 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.567046+0000", 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019248609999999999 2024-09-13T14:25:48.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:46.567097+0000", 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1103999999999997e-05 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:24053 5.7 e108/104 5:e5e33199:::benchmark_data_smithi017_147622_object24052:head v 108'1512, mlcod=108'1510)", 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:25:48.228681+0000", 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.067304118999999996, 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014081216000000001, 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:25:48.457 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228681+0000", 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228681+0000", 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228682+0000", 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0800000000000005e-07 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228711+0000", 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9241999999999999e-05 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228720+0000", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2909999999999996e-06 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228726+0000", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8730000000000004e-06 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228801+0000", 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5036000000000005e-05 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.228819+0000", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8227999999999999e-05 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.242700+0000", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013880648000000001 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:25:48.242762+0000", 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2390000000000004e-05 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.461 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.462 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.462 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:25:48.462 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:25:48.462 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:25:48.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:48.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:48.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:48.735 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:48.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:48.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:48.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:48.874 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 25665 25649 200.357 158.5 0.00944912 0.00498572 2024-09-13T14:25:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:48.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:48.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:49.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:49.140 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:49.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.177 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:25:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:49.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:49.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:49.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[103001]: pgmap v257: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2943589262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[107556]: pgmap v257: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2943589262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:49 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.457 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:49 smithi189 ceph-mon[90992]: pgmap v257: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-13T14:25:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2943589262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:49 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:25:49.558 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:49.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:49.735 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:49.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-13T14:25:49.836 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:49.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:49.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:49.875 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 28228 28212 195.892 160.188 0.00949622 0.00509859 2024-09-13T14:25:49.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:50.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:50.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:50.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:50.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:50.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:50.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:50.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[103001]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:25:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:50 smithi017 ceph-mon[107556]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:25:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:50 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:25:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:50 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:25:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:50 smithi189 ceph-mon[90992]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:25:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:25:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:25:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:25:50.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:50.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:50.656 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:50.691 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:50.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:50.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:50.875 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 30467 30451 190.295 139.938 0.00267312 0.00521385 2024-09-13T14:25:50.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:50.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:50.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:51.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:51.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:51.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:51.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:51.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:51.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:51.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:51 smithi017 ceph-mon[103001]: pgmap v259: 89 pgs: 89 active+clean; 1.3 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T14:25:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:51 smithi017 ceph-mon[107556]: pgmap v259: 89 pgs: 89 active+clean; 1.3 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T14:25:51.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:51.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:51 smithi189 ceph-mon[90992]: pgmap v259: 89 pgs: 89 active+clean; 1.3 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T14:25:51.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:51.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:51.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:51.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:51.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:51.875 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 32430 32414 184.148 122.688 0.0023702 0.00542542 2024-09-13T14:25:51.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:51.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:51.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:52.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:52.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:52.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:52.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:52.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:52.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:52.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:52.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:52.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:52.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:52.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:52.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:52.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:52.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:52.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:52.875 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 34802 34786 181.156 148.25 0.00908274 0.00551458 2024-09-13T14:25:52.914 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:52.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:25:52.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:53.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:53.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:53.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:53.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:53.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:53.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:53.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:53 smithi189 ceph-mon[90992]: pgmap v260: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:25:53.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:53 smithi017 ceph-mon[103001]: pgmap v260: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:25:53.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:53 smithi017 ceph-mon[107556]: pgmap v260: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:25:53.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:53.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:53.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:53.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:53.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:53.766 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:53.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:53.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:53.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 36839 36823 177.004 127.312 0.00487813 0.00564436 2024-09-13T14:25:53.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:53.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:53.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:54.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:54.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:54.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:54.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:54.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:54.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:54.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:25:54.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:54.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:54.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:54.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:54.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:54.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:54.877 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 38783 38767 173.039 121.5 0.0113344 0.0057739 2024-09-13T14:25:54.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:54.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:54.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:55.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:55.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:55.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:55.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:55.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:55 smithi017 ceph-mon[103001]: pgmap v261: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-09-13T14:25:55.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:55 smithi017 ceph-mon[107556]: pgmap v261: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-09-13T14:25:55.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:55.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:55 smithi189 ceph-mon[90992]: pgmap v261: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2024-09-13T14:25:55.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:55.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:55.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:55.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:55.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:55.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:55.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 40708 40692 169.524 120.312 0.0034228 0.00589321 2024-09-13T14:25:55.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:55.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:55.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:56.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:56.095 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:56.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:56.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:56.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:56.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:56.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:56.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:56.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:56.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:56.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:56.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:25:56.792 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:25:56.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:25:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:56.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:56.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 42775 42759 167.002 129.188 0.00716332 0.005983 2024-09-13T14:25:56.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:56.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:56.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:57.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:57.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:57.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:57.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:57.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:57.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:57.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:57.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[103001]: pgmap v262: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/96418929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:25:57] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[107556]: pgmap v262: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/96418929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:57.483 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:57 smithi189 ceph-mon[90992]: pgmap v262: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2024-09-13T14:25:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/96418929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:25:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:25:57.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:25:57.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:57.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:57.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:25:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:57.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:25:57.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 44850 44834 164.806 129.688 0.0116063 0.00606272 2024-09-13T14:25:57.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:57.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:57.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:58.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:58.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:58.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:25:58.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:58.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:25:58.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:58.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:58.347 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:58.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:58.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:58.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:58.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:58.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:25:58.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:25:58.866 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:25:58.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 46881 46865 162.702 126.938 0.00385371 0.0061411 2024-09-13T14:25:58.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:58.903 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:58.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:25:59.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:59.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:59.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:59.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:59.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:59.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:59.306 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:25:59 smithi017 ceph-mon[107556]: pgmap v263: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:25:59.306 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:25:59 smithi017 ceph-mon[103001]: pgmap v263: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:25:59.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:59.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:25:59.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:25:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:25:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:25:59 smithi189 ceph-mon[90992]: pgmap v263: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:25:59.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.610 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:59.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:59.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:25:59.744 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:59.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:25:59.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:25:59.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 48826 48810 160.536 121.562 0.00592052 0.0062238 2024-09-13T14:25:59.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:25:59.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:25:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:25:59.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:25:59.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:00.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:00.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:00.167 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:00.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:00.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:00.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:00.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:00.443 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:00.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:00.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:00.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:00.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:00.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:00.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:00.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:26:00.876655+0000 min lat: 0.00112043 max lat: 0.101005 avg lat: 0.00631202 2024-09-13T14:26:00.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:26:00.877 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 50680 50664 158.303 115.875 0.00335196 0.00631202 2024-09-13T14:26:00.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:00.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:00.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:01.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:01.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:01.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:01.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:01.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:01 smithi017 ceph-mon[107556]: pgmap v264: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:26:01.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:01 smithi017 ceph-mon[103001]: pgmap v264: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:26:01.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:01.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-13T14:26:01.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:01.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:01.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:01 smithi189 ceph-mon[90992]: pgmap v264: 89 pgs: 89 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:26:01.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:01.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:01.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:01.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:01.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:01.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:01.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 52517 52501 156.231 114.812 0.00714664 0.0063964 2024-09-13T14:26:01.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:01.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:01.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:02.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:02.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:02.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:02.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:02.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:02.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:02.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:02.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:02.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:02.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:02.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:02.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:02.877 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 54318 54302 154.246 112.562 0.0189056 0.00647847 2024-09-13T14:26:02.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:02.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:02.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:03.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:03.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:03.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:03.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:03.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:03.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:03.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:03.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:03 smithi017 ceph-mon[103001]: pgmap v265: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:03 smithi017 ceph-mon[107556]: pgmap v265: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:03.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:03.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:03.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:03 smithi189 ceph-mon[90992]: pgmap v265: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:03.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:03.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:03.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:03.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:03.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:03.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 55900 55884 151.839 98.875 0.00408188 0.00657614 2024-09-13T14:26:03.904 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:03.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:04.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:04.020 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:26:04.020 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:26:04.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:04.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:04.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:04.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:04.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:04.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:04.437 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:04.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:04.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:04.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:04.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:04.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:04.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:04.877 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 57698 57682 150.194 112.375 0.0102259 0.00665235 2024-09-13T14:26:04.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:04.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:04.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:26:04.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:05.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:05.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:05.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:05.280 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:05.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:05.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:05.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:05 smithi017 ceph-mon[103001]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:26:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:05 smithi017 ceph-mon[107556]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:26:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:05 smithi189 ceph-mon[90992]: pgmap v266: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:26:05.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:05.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:05.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:05.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:05.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:05.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:05.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:05.876 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 59333 59317 148.273 102.188 0.00914885 0.00673986 2024-09-13T14:26:05.899 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:05.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:06.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:06.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:06.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:06.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:06.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:06.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:06.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:06.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.536 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:06.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:06.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:06.877 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 60959 60943 146.479 101.625 0.0061639 0.00682132 2024-09-13T14:26:06.912 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:06.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:06.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:07.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:07.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:07.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:07.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:07.249 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:26:07.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[103001]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:26:07.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3702817529' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.270 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[107556]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:26:07.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3702817529' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:07.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:07.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:07.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:07 smithi189 ceph-mon[90992]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:26:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3702817529' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:07 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:07.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:07.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:07] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:26:07.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:07.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:07.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 15 62661 62646 144.99 106.438 0.0107904 0.00689264 2024-09-13T14:26:07.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:07.918 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:07.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:08.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:08.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:08.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:08.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:08.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:08.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:08.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:08.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:08.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[107556]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:08.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:08 smithi017 ceph-mon[103001]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:26:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:08 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:26:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:08 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:26:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:08 smithi189 ceph-mon[90992]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:26:08.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:08.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:08.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:08.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:08.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:08.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:08.878 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 64348 64332 143.575 105.375 0.00705593 0.00695988 2024-09-13T14:26:08.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:08.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:08.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:09.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:09.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:09.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:09.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:09.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:09.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:09.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:09.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:09.431 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:09.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:09 smithi189 ceph-mon[90992]: pgmap v269: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:26:09.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:09.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:09 smithi017 ceph-mon[103001]: pgmap v269: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:26:09.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:09 smithi017 ceph-mon[107556]: pgmap v269: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:26:09.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:09.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:09.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:09.785 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:26:09.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:26:09.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:26:09.824 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:26:09.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:09.863 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:26:09.863 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:26:09.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:09.878 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 65963 65947 142.104 100.938 0.0100403 0.00703185 2024-09-13T14:26:09.894 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:26:09.894 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:26:09.929 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:26:09.929 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:26:09.962 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:26:09.962 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:26:09.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:09.994 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:26:09.994 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:26:09.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:09.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:09.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:10.031 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:26:10.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:26:10.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:10.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:10.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:10.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:10.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:10.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1226404242' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1226404242' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1226404242' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:10.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:10.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:10.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:10.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:10.787 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:10.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:10.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:10.878 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 67532 67516 140.636 98.0625 0.0084909 0.00710495 2024-09-13T14:26:10.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:10.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:10.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:11.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:11.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:11.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:11.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:11.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-13T14:26:11.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:11.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:11.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:26:11.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:11.506 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:11.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:11 smithi017 ceph-mon[107556]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:26:11.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:11 smithi017 ceph-mon[103001]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:26:11.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:11.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:11.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:11 smithi189 ceph-mon[90992]: pgmap v270: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:26:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:11.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:11.878 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 69024 69008 139.106 93.25 0.00370829 0.00718372 2024-09-13T14:26:11.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:11.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:11.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:12.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:12.068 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:12.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:12.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:12.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:12.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:12.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:12.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:12.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:12.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:12.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:12.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:12.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:12.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:12.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:12.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:12.878 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 70518 70502 137.677 93.375 0.0128128 0.00725821 2024-09-13T14:26:12.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:12.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:12.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:12.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:13.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:13.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:13.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:13.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:13.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:13.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:13.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[103001]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[107556]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:13 smithi189 ceph-mon[90992]: pgmap v271: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:26:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:13.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:13.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:13.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:13.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:13.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:13.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:13.861 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:13.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:13.879 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 72041 72025 136.389 95.1875 0.00760607 0.00732685 2024-09-13T14:26:13.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:26:13.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:14.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:14.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:14.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:14.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:14.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:14.364 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:14.376 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:14.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:14.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:14.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:14.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:14.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:14.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:14.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:14.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:14.879 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 73475 73459 135.014 89.625 0.00243202 0.00740039 2024-09-13T14:26:14.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:14.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:14.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:14.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:15.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:15.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:15.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:15.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:15.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:15.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:15 smithi017 ceph-mon[103001]: pgmap v272: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:26:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:15 smithi017 ceph-mon[107556]: pgmap v272: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:26:15.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:15 smithi189 ceph-mon[90992]: pgmap v272: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:26:15.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:15.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:15.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:15.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:15.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:15.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:15.880 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 74804 74788 133.522 83.0625 0.01486 0.00748449 2024-09-13T14:26:15.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:15.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:16.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:16.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:16.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:16.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:16.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:16.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:16.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:16.477 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:16.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:16.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:16.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:16.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:16.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:16.882 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 76232 76216 132.292 89.25 0.00885303 0.00755382 2024-09-13T14:26:16.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:16.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:16.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:17.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:17.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:17.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:17 smithi017 ceph-mon[103001]: pgmap v273: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:26:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:17 smithi017 ceph-mon[107556]: pgmap v273: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:26:17.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:17.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:17.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:17.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:17.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:17 smithi189 ceph-mon[90992]: pgmap v273: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:26:17.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:17.562 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:17.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:17.673 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:17] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:26:17.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:17.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:17.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:17.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 77529 77513 130.908 81.0625 0.0118254 0.00763411 2024-09-13T14:26:17.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:17.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:17.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:18.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:18.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:18.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:18.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:18.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:18.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:18.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:18.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:18.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:18.573 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:18.589 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:26:18.589 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:26:18.589 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:26:18.589 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:26:18.589 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:26:18.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:18.674 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:18.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:18.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:18.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:18.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 78954 78938 129.806 89.0625 0.00640167 0.00769964 2024-09-13T14:26:18.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:18.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:18.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:19.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:19.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:19.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:19.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:19.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:19.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:19.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:19.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:19.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:19.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:19 smithi017 ceph-mon[103001]: pgmap v274: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-09-13T14:26:19.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:19 smithi017 ceph-mon[107556]: pgmap v274: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-09-13T14:26:19.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:19 smithi189 ceph-mon[90992]: pgmap v274: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-09-13T14:26:19.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:19.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:19.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:19.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:19.800 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:19.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:19.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 80356 80340 128.725 87.625 0.00627029 0.00776393 2024-09-13T14:26:19.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:19.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:19.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:19.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:20.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:20.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:20.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:20.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:20.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:20.359 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:20.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:20.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:20.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:20.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:20.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:20.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:20.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:20.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:26:20.881626+0000 min lat: 0.00112043 max lat: 0.107612 avg lat: 0.00782783 2024-09-13T14:26:20.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:26:20.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 81738 81722 127.666 86.375 0.0133684 0.00782783 2024-09-13T14:26:20.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:20.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:21.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:21.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:21.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:21.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:21 smithi017 ceph-mon[103001]: pgmap v275: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:21 smithi017 ceph-mon[107556]: pgmap v275: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:21.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:21.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:21.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:21.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:21 smithi189 ceph-mon[90992]: pgmap v275: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:21.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:21.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:21.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:21.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:21.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:21.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:21.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:21.882 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 83125 83109 126.666 86.6875 0.0154147 0.00789054 2024-09-13T14:26:21.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:22.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:22.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:22.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:22.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:22.266 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:22.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:26:22.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:22.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:22.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:22.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:22.577 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:22.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:22.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:22.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:22.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:22.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:22.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:22.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 84466 84450 125.646 83.8125 0.0186212 0.00795423 2024-09-13T14:26:22.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:23.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:23.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:23.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:23.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:23.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:23.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:23 smithi017 ceph-mon[103001]: pgmap v276: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:23.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:23 smithi017 ceph-mon[107556]: pgmap v276: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:23.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:23 smithi189 ceph-mon[90992]: pgmap v276: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:23.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:23.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:23.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:23.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:23.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:23.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:23.881 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 85638 85622 124.427 73.25 0.0224159 0.00803216 2024-09-13T14:26:23.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:24.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:24.132 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:26:24.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:24.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:24.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:24.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:24 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3786722459' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:24 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:24 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3786722459' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:24 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:24 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3786722459' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:24 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:26:24.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:24.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-13T14:26:24.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:24.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:24.746 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:24.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:24.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:24.885 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 86769 86753 123.195 70.6875 0.0065428 0.00811215 2024-09-13T14:26:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:24.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:24.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:25.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:25.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:25.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.204 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:25.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:25.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:25.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:25.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:25.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[103001]: pgmap v277: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-09-13T14:26:25.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:25.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:26:25.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[103001]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:26:25.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[107556]: pgmap v277: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-09-13T14:26:25.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:25.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:26:25.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:25 smithi017 ceph-mon[107556]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:26:25.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:25.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:25 smithi189 ceph-mon[90992]: pgmap v277: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-09-13T14:26:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:25 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:26:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:25 smithi189 ceph-mon[90992]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:26:25.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:25.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:25.721 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:25.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:25.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:25.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:25.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:25.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:25.885 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 88048 88032 122.234 79.9375 0.011137 0.00817627 2024-09-13T14:26:25.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:26.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:26.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:26.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.294 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:26.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:26.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:26.428 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:26.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:26.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:26.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:26.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:26.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:26.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 89304 89288 121.283 78.5 0.0036027 0.00823863 2024-09-13T14:26:26.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:26.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:26.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:27.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:27.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:27.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:27.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:27.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:27.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:27] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-09-13T14:26:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[103001]: pgmap v279: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:26:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[107556]: pgmap v279: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:26:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:27 smithi189 ceph-mon[90992]: pgmap v279: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:26:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:27.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:27.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:27.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:27.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:27.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:27.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 90512 90496 120.309 75.5 0.0164019 0.00830724 2024-09-13T14:26:27.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:27.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:27.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:28.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:28.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:28.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:28.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:28.236 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:28.269 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:26:28.270 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:26:28.312 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:26:28.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:26:28.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:28.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:26:28.347 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:26:28.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:28.381 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:26:28.382 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:26:28.417 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:26:28.417 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:26:28.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:26:28.447 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:26:28.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:28.483 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:26:28.483 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:26:28.513 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:26:28.514 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:26:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:28.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:28.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:28.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:28.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:28.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:28.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 91748 91732 119.412 77.25 0.0160868 0.00836918 2024-09-13T14:26:28.895 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:28.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:28.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:29.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:29.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:29.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:29.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:29.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:29.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:29.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:29 smithi017 ceph-mon[103001]: pgmap v280: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3688644323' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:29 smithi017 ceph-mon[107556]: pgmap v280: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3688644323' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.471 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:29 smithi189 ceph-mon[90992]: pgmap v280: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3688644323' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:29.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:29.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:29.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:29.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:29.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:29.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 93020 93004 118.597 79.5 0.01363 0.0084269 2024-09-13T14:26:29.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:29.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:30.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:26:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:30.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:30.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:30.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:30.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:30.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:30.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:30.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:30.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:30.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:30.618 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:30.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:30.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:30.834 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:30.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:30.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:30.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 94250 94234 117.763 76.875 0.0154627 0.00848648 2024-09-13T14:26:30.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:31.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:31.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:31.151 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:31.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:31 smithi017 ceph-mon[103001]: pgmap v281: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:31.178 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:31 smithi017 ceph-mon[107556]: pgmap v281: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:31.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:31.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:31.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:31.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:31 smithi189 ceph-mon[90992]: pgmap v281: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:26:31.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:31.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:31.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:31.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:31.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:31.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:31.850 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:26:31.851 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:26:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:31.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:31.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 95472 95456 116.951 76.375 0.00870431 0.00854612 2024-09-13T14:26:31.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:31.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:31.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:31.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:32.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:26:32.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:32.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:32.234 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:32.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:32.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:32.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:32.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:32.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:32.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:32.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:32.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:26:32.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:32.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:32.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:32.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:32.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 96651 96635 116.119 73.6875 0.0173553 0.00860666 2024-09-13T14:26:32.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:32.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:32.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:33.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:33.088 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:33.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:33.144 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:33.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:33.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:33.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:33.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:33.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:33 smithi017 ceph-mon[103001]: pgmap v282: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:26:33.415 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:33 smithi017 ceph-mon[107556]: pgmap v282: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:26:33.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:33 smithi189 ceph-mon[90992]: pgmap v282: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:26:33.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:33.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:33.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:33.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:33.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:33.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 97815 97799 115.301 72.75 0.01122 0.00866803 2024-09-13T14:26:33.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:33.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:33.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:34.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:34.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:34.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:34.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:34.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:34.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:34.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:34.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:34.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:34.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:34.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:34.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:34.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:34.886 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 99008 98992 114.546 74.5625 0.00737055 0.00872456 2024-09-13T14:26:34.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:34.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:34.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:35.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:35.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:35.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:35.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:35.300 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:35.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:35.431 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:35 smithi017 ceph-mon[103001]: pgmap v283: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T14:26:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:35 smithi017 ceph-mon[107556]: pgmap v283: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T14:26:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:35 smithi189 ceph-mon[90992]: pgmap v283: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T14:26:35.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:35.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-13T14:26:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:35.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:35.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:35.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:35.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:35.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:35.853 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:35.891 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 100175 100159 113.79 72.9375 0.016171 0.00878318 2024-09-13T14:26:35.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:35.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:35.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:36.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.137 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:36.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:36.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:36.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:36.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:36.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:36.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:36.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:36.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:36.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:36.787 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:36.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:36.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 101346 101330 113.065 73.1875 0.0132087 0.00883934 2024-09-13T14:26:36.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:36.973 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:36.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:37.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:37.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:37.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:37.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:37.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:37 smithi017 ceph-mon[103001]: pgmap v284: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:26:37.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:37.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:37 smithi017 ceph-mon[107556]: pgmap v284: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:26:37.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:37.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:37.397 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:37.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:37 smithi189 ceph-mon[90992]: pgmap v284: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:26:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:26:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:37.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:37.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:37.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:37] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-09-13T14:26:37.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:37.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:37.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 102288 102272 112.114 58.875 0.00991482 0.00891497 2024-09-13T14:26:37.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:37.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:37.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:38.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:38.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:38.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:38.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:38.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:38.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:38.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:38.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:38.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:38.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:38.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:38.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:38.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 103341 103325 111.316 65.8125 0.0153532 0.00897824 2024-09-13T14:26:38.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:38.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:38.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:38.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:39.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:39.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:39.172 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:26:39.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:39.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:39 smithi189 ceph-mon[90992]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-13T14:26:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:39.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:39.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:39 smithi017 ceph-mon[103001]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-13T14:26:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:39 smithi017 ceph-mon[107556]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-09-13T14:26:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:26:39.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:39.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:39.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:39.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:39.674 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:39.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:39.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:39.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 104414 104398 110.566 67.0625 0.018699 0.00903956 2024-09-13T14:26:39.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:39.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:39.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:40.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:40.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:40.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:40.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:40.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:40.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:40.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:40.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[107556]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:26:40.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:40.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:40.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:40 smithi017 ceph-mon[103001]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:26:40.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:40.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:40 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3440164594' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:26:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:40 smithi189 ceph-mon[90992]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:26:40.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:40.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:40.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:40.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:40.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:40.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:26:40.887439+0000 min lat: 0.00112043 max lat: 0.108415 avg lat: 0.00909944 2024-09-13T14:26:40.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:26:40.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 105485 105469 109.839 66.9375 0.0178424 0.00909944 2024-09-13T14:26:40.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:40.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:40.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:41.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:41.062 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:41.065 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:26:41.065 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:26:41.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:41.101 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:26:41.101 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:26:41.142 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:26:41.142 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:26:41.175 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:26:41.176 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:26:41.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:41.209 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:26:41.209 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:26:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:41.242 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:26:41.242 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:26:41.274 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:26:41.275 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:26:41.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:26:41.304 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:26:41.305 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:41 smithi017 ceph-mon[103001]: pgmap v287: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:26:41.305 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3834443152' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:41.305 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:41 smithi017 ceph-mon[107556]: pgmap v287: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:26:41.305 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3834443152' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:41.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:41.458 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:41 smithi189 ceph-mon[90992]: pgmap v287: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:26:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3834443152' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:41.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:41.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:41.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:41.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:41.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:41.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 106465 106449 109.042 61.25 0.0118279 0.00916504 2024-09-13T14:26:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:41.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:41.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:42.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:42.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:42.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:42.173 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:42.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:42.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:42.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:26:42.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:42.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:42.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:42.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:42.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:42.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:42.684 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:26:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:26:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:107206 5.b 5:d453853c:::benchmark_data_smithi017_147622_object107205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:26:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:26:42.665138+0000", 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.016972930000000001, 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017005945000000001, 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 107206 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665138+0000", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665138+0000", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665140+0000", 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.916e-06 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665175+0000", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4545000000000002e-05 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665188+0000", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3638000000000001e-05 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.665196+0000", 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9480000000000008e-06 2024-09-13T14:26:42.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.667348+0000", 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021518800000000001 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.667480+0000", 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131538 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:26:42.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.667618+0000", 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138777 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.671217+0000", 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035981310000000001 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:107219 5.d 5:be5d1acb:::benchmark_data_smithi017_147622_object107218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:26:42.689 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:26:42.676782+0000", 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0053295440000000003, 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054403799999999999, 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 107219 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:26:42.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676782+0000", 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676782+0000", 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676783+0000", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.548e-06 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676804+0000", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0939999999999999e-05 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676828+0000", 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4156e-05 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676836+0000", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6480000000000003e-06 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.677011+0000", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017487700000000001 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.677148+0000", 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136737 2024-09-13T14:26:42.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.677300+0000", 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015273700000000001 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:107223 5.b 5:d57d134c:::benchmark_data_smithi017_147622_object107222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:26:42.676864+0000", 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0052473650000000004, 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054228999999999996, 2024-09-13T14:26:42.694 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 107223 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676864+0000", 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676864+0000", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676864+0000", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9999999999999997e-07 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676941+0000", 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6280999999999997e-05 2024-09-13T14:26:42.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676950+0000", 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4909999999999994e-06 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.676956+0000", 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9070000000000004e-06 2024-09-13T14:26:42.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.678098+0000", 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011419830000000001 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.678213+0000", 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115433 2024-09-13T14:26:42.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.678339+0000", 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012567200000000001 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:107225 5.a 5:5fb75afc:::benchmark_data_smithi017_147622_object107224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:26:42.680000+0000", 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0021114770000000001, 2024-09-13T14:26:42.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023572789999999999, 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 107225 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680000+0000", 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680000+0000", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680001+0000", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.575e-06 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680023+0000", 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1880000000000001e-05 2024-09-13T14:26:42.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680040+0000", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7197000000000002e-05 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680049+0000", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6619999999999994e-06 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680172+0000", 2024-09-13T14:26:42.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012341699999999999 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680312+0000", 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013991200000000001 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680434+0000", 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012215800000000001 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:107226 5.6 5.448d4f16 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:26:42.680073+0000", 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0020382320000000001, 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023557840000000001, 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 107226 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:26:42.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680073+0000", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680073+0000", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680073+0000", 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7899999999999999e-07 2024-09-13T14:26:42.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680093+0000", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9797999999999999e-05 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680101+0000", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5340000000000002e-06 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:26:42.680106+0000", 2024-09-13T14:26:42.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6289999999999998e-06 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 5 2024-09-13T14:26:42.707 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:26:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:42.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:42.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:42.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:42.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 107393 107377 108.219 58 0.00428786 0.00923074 2024-09-13T14:26:42.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:42.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:42.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:43.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:43.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:43.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:43.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:43.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:43.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:43 smithi017 ceph-mon[103001]: pgmap v288: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:26:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:43 smithi017 ceph-mon[107556]: pgmap v288: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:26:43.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:43 smithi189 ceph-mon[90992]: pgmap v288: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:26:43.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:43.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:43.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:43.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:43.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:43.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 108492 108476 107.591 68.6875 0.00880115 0.00928985 2024-09-13T14:26:43.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:43.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:43.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:43.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:44.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:44.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:44.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:44.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:44.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:44.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:44.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:44.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:44.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:44.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:44.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:44.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:44.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:44.888 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 109440 109424 106.836 59.25 0.0189982 0.00935507 2024-09-13T14:26:44.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:44.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:45.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:45.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:45.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:45.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:45.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:45.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:45 smithi017 ceph-mon[103001]: pgmap v289: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:26:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:45 smithi017 ceph-mon[107556]: pgmap v289: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:26:45.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:45.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:45.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:45 smithi189 ceph-mon[90992]: pgmap v289: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:26:45.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:45.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:45.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:45.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:45.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:45.887 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 110490 110474 106.202 65.625 0.013439 0.00941072 2024-09-13T14:26:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:45.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:45.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:46.020 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.060 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:46.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:46.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:46.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:46.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:26:46.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:46.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:46.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:46.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:46.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:46.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:46.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:46.755 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:46.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:46.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:46.888 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 111480 111464 105.53 61.875 0.0210773 0.00947018 2024-09-13T14:26:46.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:46.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:46.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:47.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:47.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:47.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:47.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:47.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:47.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:47.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:47.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:47.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:47.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[103001]: pgmap v290: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:26:47.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.550 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:47] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:26:47.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[107556]: pgmap v290: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:26:47.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:47.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:47.754 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:47 smithi189 ceph-mon[90992]: pgmap v290: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:26:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:26:47.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:47.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:26:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:47.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:47.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 112532 112516 104.934 65.75 0.0250901 0.00952463 2024-09-13T14:26:47.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:47.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:47.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:48.082 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:48.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:48.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:48.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:48.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:48.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:48.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:48.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:48.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:48.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:48.659 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:48.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:48.794 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:48.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:48.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 113579 113563 104.353 65.4375 0.0125193 0.00957774 2024-09-13T14:26:48.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:48.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:48.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:49.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:49.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:49.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:49.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:49.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:49.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:49.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:49.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:49.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:49.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:49 smithi017 ceph-mon[103001]: pgmap v291: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:26:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:49 smithi017 ceph-mon[107556]: pgmap v291: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:26:49.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:49 smithi189 ceph-mon[90992]: pgmap v291: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:26:49.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:49.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:49.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 114622 114606 103.785 65.1875 0.0101324 0.00963041 2024-09-13T14:26:49.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:49.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:49.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:50.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:50.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:50.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:50.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:50.290 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:50.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:50.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:50.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:26:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:26:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:26:50.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:50.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:50.737 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:50.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:50.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:50.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:50.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 115655 115639 103.225 64.5625 0.012784 0.00968308 2024-09-13T14:26:50.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:51.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:51.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:51.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:51.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:51.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:51.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-13T14:26:51.403 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:51.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:51.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:51.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:51.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:51 smithi017 ceph-mon[103001]: pgmap v292: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:26:51.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:51 smithi017 ceph-mon[107556]: pgmap v292: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:26:51.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:51.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:51 smithi189 ceph-mon[90992]: pgmap v292: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-09-13T14:26:51.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:51.861 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:51.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:51.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 116671 116655 102.666 63.5 0.0105418 0.0097357 2024-09-13T14:26:51.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:52.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:52.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:52.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:52.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:52.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:52.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:52.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:52.431 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:26:52.431 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:26:52.431 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:26:52.431 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:26:52.431 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:26:52.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:52.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:52.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:52.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:52.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:52.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:52.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:52.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:52.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 117677 117661 102.113 62.875 0.0172664 0.00978829 2024-09-13T14:26:52.896 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:52.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:53.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:53.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:53.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:53.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:53.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:53.386 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:26:53.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:53.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:53.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:53.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:53.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:26:53.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[103001]: pgmap v293: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2961962000' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[107556]: pgmap v293: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2961962000' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:53 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:53 smithi189 ceph-mon[90992]: pgmap v293: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:26:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2961962000' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:53 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:26:53.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:53.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:53.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 118658 118642 101.554 61.3125 0.00729108 0.00984198 2024-09-13T14:26:53.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:53.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:53.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:54.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:54.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:54.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:54.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:54.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:54.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:54.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:54.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:54.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:54.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:54.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:54.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:26:54.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[103001]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:26:54.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:54.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:26:54.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:54 smithi017 ceph-mon[107556]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:26:54.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:54 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:26:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:54 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:26:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:54 smithi189 ceph-mon[90992]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:26:54.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:54.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:54.890 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 119612 119596 100.987 59.625 0.0244789 0.00989663 2024-09-13T14:26:54.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:54.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:54.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:54.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:55.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:55.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:55.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:55.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:55.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:55.356 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.396 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:55.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:55 smithi017 ceph-mon[103001]: pgmap v295: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:26:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:55 smithi017 ceph-mon[107556]: pgmap v295: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:26:55.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:55.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:55.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:55.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:55 smithi189 ceph-mon[90992]: pgmap v295: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:26:55.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:55.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:55.869 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:55.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 120566 120550 100.431 59.625 0.00813969 0.00995246 2024-09-13T14:26:55.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:55.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:55.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:56.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:56.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:56.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:56.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:56.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:56.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:56.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:56.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:56.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:26:56.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:56.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:56.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:56.668 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:56.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:56.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:56.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:56.895 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 121494 121478 99.872 58 0.0136077 0.0100081 2024-09-13T14:26:56.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:56.943 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:56.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:57.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:57.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:57.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:57.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:57.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:57.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:57 smithi017 ceph-mon[103001]: pgmap v296: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:26:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:26:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:26:57] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:26:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:57 smithi017 ceph-mon[107556]: pgmap v296: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:26:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:57.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:57.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:57.654 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:26:57.654 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:26:57.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:57.691 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:26:57.691 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:26:57.728 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:26:57.728 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:26:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:57 smithi189 ceph-mon[90992]: pgmap v296: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:26:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:26:57.762 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:26:57.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:26:57.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:57.791 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:26:57.792 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:26:57.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:57.828 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:26:57.829 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:26:57.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:26:57.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:26:57.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:26:57.892 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:26:57.895 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 122467 122451 99.3648 60.8125 0.0146769 0.0100586 2024-09-13T14:26:57.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:57.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:57.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:57.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:58.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:26:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:58.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:26:58.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:58.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:58.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:58.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:58.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:58.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:58.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:58.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:58.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:58.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:58.649 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:58.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1069364782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1069364782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:58.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:26:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1069364782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:26:58.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:26:58.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:26:58.895 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 123440 123424 98.8706 60.8125 0.00553568 0.0101091 2024-09-13T14:26:58.934 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:58.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:58.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:26:59.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:26:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:59.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:26:59.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:59.212 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:59.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:26:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:59.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:26:59.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:59.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:59.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:26:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:26:59.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:26:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:26:59 smithi017 ceph-mon[107556]: pgmap v297: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-09-13T14:26:59.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:26:59 smithi017 ceph-mon[103001]: pgmap v297: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-09-13T14:26:59.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:26:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:26:59 smithi189 ceph-mon[90992]: pgmap v297: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-09-13T14:26:59.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:26:59.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:26:59.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 124433 124417 98.4037 62.0625 0.0105496 0.0101571 2024-09-13T14:26:59.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:26:59.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:59.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:26:59.977 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:00.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:00.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:00.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:00.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:00.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:00.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:00.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:00.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:00.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:00.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:00.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:00.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:00.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:00.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:00.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:00.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:00.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:27:00.895977+0000 min lat: 0.00112043 max lat: 0.108415 avg lat: 0.010207 2024-09-13T14:27:00.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:27:00.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 125394 125378 97.9245 60.0625 0.0136537 0.010207 2024-09-13T14:27:00.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:01.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:01.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:01.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:01.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.296 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:01.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:01.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:01.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:01.485 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:01 smithi017 ceph-mon[103001]: pgmap v298: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:27:01.486 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:01 smithi017 ceph-mon[107556]: pgmap v298: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:27:01.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:01.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:01.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:01.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:01 smithi189 ceph-mon[90992]: pgmap v298: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:27:01.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:01.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:01.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:01.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 126263 126247 97.386 54.3125 0.00957025 0.0102632 2024-09-13T14:27:01.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:02.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:02.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:02.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:02.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:02.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:02.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:02.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:02.561 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:02.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:02.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:02.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:02.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-13T14:27:02.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:02.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:02.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:02.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 15 127183 127168 96.9004 57.5625 0.016012 0.0103151 2024-09-13T14:27:02.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:03.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:03.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:03.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:03.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:03.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:03.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:03.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:03.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:03.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:03.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:03 smithi017 ceph-mon[107556]: pgmap v299: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-09-13T14:27:03.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:03 smithi017 ceph-mon[103001]: pgmap v299: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-09-13T14:27:03.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:03.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:03.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:03 smithi189 ceph-mon[90992]: pgmap v299: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-09-13T14:27:03.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:03.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:03.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 128165 128149 96.4716 61.3125 0.010501 0.0103607 2024-09-13T14:27:03.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:03.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:03.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:04.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:04.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:04.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:04.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:04.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:04.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:04.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:04.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:04.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:04.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:04.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:04.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:04.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 129159 129143 96.0628 62.125 0.0134775 0.0104045 2024-09-13T14:27:04.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:04.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:04.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:05.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:05.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:05.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:05.263 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:05.365 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:05.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:05.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:05.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:05.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:27:05.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:05 smithi017 ceph-mon[103001]: pgmap v300: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T14:27:05.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:05 smithi017 ceph-mon[107556]: pgmap v300: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T14:27:05.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:05.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:05 smithi189 ceph-mon[90992]: pgmap v300: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T14:27:05.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:05.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:05.897 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 130079 130063 95.6088 57.5 0.00972357 0.0104542 2024-09-13T14:27:05.915 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:05.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:05.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:06.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:06.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:06.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:06.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:06.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:06.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:06.375 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:27:06.375 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:27:06.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:06.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:06.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:06.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:06.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:06.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:06.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:06.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:06.896 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 131019 131003 95.1802 58.75 0.0167199 0.0105015 2024-09-13T14:27:06.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:06.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:06.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:06.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:07.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:07.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:07.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:07.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:07.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:07.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:07.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:07.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:07.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:07.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:07.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:07] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-09-13T14:27:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:07 smithi017 ceph-mon[103001]: pgmap v301: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:27:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:07 smithi017 ceph-mon[107556]: pgmap v301: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:27:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:07 smithi189 ceph-mon[90992]: pgmap v301: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:27:07.777 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:07.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:07.897 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 131975 131959 94.773 59.75 0.0102529 0.0105467 2024-09-13T14:27:07.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:07.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:07.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:07.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:08.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:08.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:08.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:08.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:08.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:08.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:08.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:27:08.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:08.466 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:08.493 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:08.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:08.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:08.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:08.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:08.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:08.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:08.897 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 132816 132800 94.2924 52.5625 0.0171665 0.0106002 2024-09-13T14:27:08.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:08.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:09.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:09.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:09.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:09.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:09.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:09.323 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:27:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:09.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:09.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:09.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:09.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:09.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[103001]: pgmap v302: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:27:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2000897356' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[107556]: pgmap v302: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:27:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2000897356' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:09 smithi189 ceph-mon[90992]: pgmap v302: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:27:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2000897356' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:09.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:09.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:09.897 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 133704 133688 93.8566 55.5 0.0230005 0.0106495 2024-09-13T14:27:09.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:09.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:10.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:10.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:10.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:10.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:10.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:10.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:10.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:10.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:10.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:10.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:10.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:27:10.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:10.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:10.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:10.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:10.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:10.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:10.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0097 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 134626 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 93.4802 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 42.3238 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 248.312 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 52.5625 2024-09-13T14:27:10.898 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1495 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 677.181 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3973 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 841 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0106934 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.00739884 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.108415 2024-09-13T14:27:10.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00112043 2024-09-13T14:27:10.902 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:27:10.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:11.005 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:11.107 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:11.140 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:11.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:11.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:11.255 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:27:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=cleanup t=2024-09-13T14:27:10.815976532Z level=info msg="Completed cleanup jobs" duration=13.212556ms 2024-09-13T14:27:11.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:11.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:11.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:11.416 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:11.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:11.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:11.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:11.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:11 smithi017 ceph-mon[103001]: pgmap v304: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T14:27:11.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:11 smithi017 ceph-mon[107556]: pgmap v304: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T14:27:11.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:11.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:11 smithi189 ceph-mon[90992]: pgmap v304: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T14:27:11.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:11.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:11.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:11.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:11.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:12.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:12.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:12.155 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:12.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:12.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:12.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:12.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:12.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:12.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:12.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:12.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:27:12.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:12.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:12.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:12.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:12.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:12.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:12.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:13.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:13.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:13.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:13.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:13.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:13.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:13.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[107556]: pgmap v305: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[103001]: pgmap v305: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:13.477 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:13 smithi189 ceph-mon[90992]: pgmap v305: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T14:27:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:13.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:13.610 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:13.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:13.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:13.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:13.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:13.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:13.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:13.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:14.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:14.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:14.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:14.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:14.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:14.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:14.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:14.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:14.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:14.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:14.617 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:27:14.617 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:27:14.648 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:27:14.648 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:27:14.680 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:27:14.680 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:27:14.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:14.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:14.722 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:27:14.722 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:27:14.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:27:14.764 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:27:14.793 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:27:14.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:27:14.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:14.825 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:27:14.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:27:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:14.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:14.862 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:27:14.863 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:27:14.950 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:14.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:14.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:15.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:15.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:15.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:15.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:15.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:15.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:15 smithi017 ceph-mon[107556]: pgmap v306: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 939 op/s 2024-09-13T14:27:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/468713515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:15 smithi017 ceph-mon[103001]: pgmap v306: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 939 op/s 2024-09-13T14:27:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/468713515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:15.464 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:15 smithi189 ceph-mon[90992]: pgmap v306: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 939 op/s 2024-09-13T14:27:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/468713515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:15.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:15.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:15.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:15.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:15.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:15.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:15.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:15.955 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:15.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:16.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:16.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:16.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:16.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:16.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:16.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:16.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:16.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:16.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:16.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:16.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:16.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:16.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:16.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:16.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:16.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:17.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:17.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:17.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:17.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:17.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:17 smithi017 ceph-mon[107556]: pgmap v307: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 940 op/s 2024-09-13T14:27:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:17 smithi017 ceph-mon[103001]: pgmap v307: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 940 op/s 2024-09-13T14:27:17.295 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:17.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:17.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:17.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:17 smithi189 ceph-mon[90992]: pgmap v307: 89 pgs: 89 active+clean; 8.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 940 op/s 2024-09-13T14:27:17.560 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:17] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-09-13T14:27:17.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:17.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:17.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:17.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:17.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:17.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:17.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:17.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:27:17.971 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:27:17.971 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:141018 5.0 5:01c8ff98:::benchmark_data_smithi017_147622_object6389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:27:17.551847+0000", 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.005445002, 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054645989999999997, 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:27:17.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 141018 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551847+0000", 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551847+0000", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551849+0000", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6220000000000001e-06 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551854+0000", 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7600000000000002e-06 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551867+0000", 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3331e-05 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551875+0000", 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8879999999999997e-06 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.551936+0000", 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0643000000000002e-05 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.552049+0000", 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011317800000000001 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.552162+0000", 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011327099999999999 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:27:17.556666+0000", 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045040210000000004 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:27:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 1 2024-09-13T14:27:17.978 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:27:17.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:18.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:18.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:18.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:18.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:18.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:18.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:18.526 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:18.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:18.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:18.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:18.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:27:18.788 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:18.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:18.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:18.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:18.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:19.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:19.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:19.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:19.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:19.314 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:27:19.315 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:27:19.315 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-13T14:27:19.315 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:27:19.315 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:27:19.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-13T14:27:19.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:19.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:19 smithi017 ceph-mon[103001]: pgmap v308: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 937 op/s 2024-09-13T14:27:19.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:19 smithi017 ceph-mon[107556]: pgmap v308: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 937 op/s 2024-09-13T14:27:19.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:19.384 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:19.393 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:19.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:19 smithi189 ceph-mon[90992]: pgmap v308: 89 pgs: 89 active+clean; 8.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 937 op/s 2024-09-13T14:27:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:19.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:19.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:19.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:19.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:19.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:19.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:19.927 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:19.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:19.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:20.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:20.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:20.130 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:27:20.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:20.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:20.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:20.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1336239972' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:20 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1336239972' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:20 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1336239972' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:20 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:27:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:20.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:20.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:20.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:20.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:20.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:20.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:20.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:20.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:21.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:21.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:21.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:21.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:21.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:21.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:21.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[103001]: pgmap v309: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 21 MiB/s wr, 893 op/s 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[103001]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[107556]: pgmap v309: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 21 MiB/s wr, 893 op/s 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:27:21.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:21 smithi017 ceph-mon[107556]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:21.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:21.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:21 smithi189 ceph-mon[90992]: pgmap v309: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 21 MiB/s wr, 893 op/s 2024-09-13T14:27:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:21 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:21.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:27:21.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:21 smithi189 ceph-mon[90992]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:21.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:21.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:21.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:21.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:21.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:21.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:21.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:21.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:22.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:22.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:22.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:22.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:22.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:22.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:22.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:22.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:22.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:22.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:22.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:22.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:22.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:22.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:22.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:22.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:23.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:23.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:23.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:23.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:23.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:23.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:23 smithi017 ceph-mon[103001]: pgmap v311: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 935 op/s 2024-09-13T14:27:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:23 smithi017 ceph-mon[107556]: pgmap v311: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 935 op/s 2024-09-13T14:27:23.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:23.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:23 smithi189 ceph-mon[90992]: pgmap v311: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 935 op/s 2024-09-13T14:27:23.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:23.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:23.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:23.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:23.796 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:23.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:23.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:23.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:23.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:24.032 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:24.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:24.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:24.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:24.295 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:24.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:27:24.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:24.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:27:24.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:24.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:24.606 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:24.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:24.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:24.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:24.874 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:24.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:24.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:25.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:25.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:27:25.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:25.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:25.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:25.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:25.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:25 smithi017 ceph-mon[103001]: pgmap v312: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 938 op/s 2024-09-13T14:27:25.313 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:25 smithi017 ceph-mon[107556]: pgmap v312: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 938 op/s 2024-09-13T14:27:25.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:27:25.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:27:25.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.428 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:25.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:25.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:27:25.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:27:25.469 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:27:25.469 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:27:25.503 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:27:25.503 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:27:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:25 smithi189 ceph-mon[90992]: pgmap v312: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 938 op/s 2024-09-13T14:27:25.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:25.538 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:27:25.539 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:27:25.562 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:25.571 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:27:25.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:27:25.605 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:27:25.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:27:25.646 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:27:25.646 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:27:25.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:25.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:25.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:25.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:25.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:25.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:25.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:26.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:26.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:26.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:26.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:26.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:26.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:26.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:26.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:26.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/35680284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/35680284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/35680284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:26.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:26.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:26.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:26.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:26.681 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:26.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:26.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:26.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:26.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:26.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:26.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:27.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:27.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:27.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:27.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:27.350 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.397 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:27.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:27] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:27:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[103001]: pgmap v313: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:27:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[107556]: pgmap v313: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:27:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:27.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:27 smithi189 ceph-mon[90992]: pgmap v313: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:27:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:27:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.536 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:27.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:27.617 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:27.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:27.686 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:27.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:27.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:27.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:27.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:27.948 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:27.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:27.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:27.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:28.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:28.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:28.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:28.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:28.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:28.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:28.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:28.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:28.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:28.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:28.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:28.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:28.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:27:28.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:28.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:28.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:29.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:29.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:29.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:29.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:29.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:29.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:29.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:29 smithi017 ceph-mon[103001]: pgmap v314: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-13T14:27:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:29 smithi017 ceph-mon[107556]: pgmap v314: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-13T14:27:29.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:29.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:29 smithi189 ceph-mon[90992]: pgmap v314: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-13T14:27:29.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:29.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:29.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:29.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:29.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:29.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:29.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:29.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:29.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:29.983 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:27:29.983 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-13T14:27:29.983 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:27:30.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:30.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:30.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:30.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:30.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:30.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:30.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:30.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:30.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:30.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:30.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:30.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:30.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:30.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:30.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:30.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:30.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:31.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:31.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:31.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.198 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:31.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:31 smithi189 ceph-mon[90992]: pgmap v315: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:27:31.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:31.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:31.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:31 smithi017 ceph-mon[103001]: pgmap v315: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:27:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:31 smithi017 ceph-mon[107556]: pgmap v315: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-09-13T14:27:31.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:31.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:31.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:31.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:31.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:31.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:31.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:31.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:31.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:31.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:31.958 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:31.982 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:32.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:32.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:32.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:32.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:32.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:32.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:32.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:32.530 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:32.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:32.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:32.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:32.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:32.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:32.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:32.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:33.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:33.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.143 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:33.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:33.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:33.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:33.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:33.421 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:33 smithi017 ceph-mon[107556]: pgmap v316: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:27:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:33 smithi017 ceph-mon[103001]: pgmap v316: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:27:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:33 smithi189 ceph-mon[90992]: pgmap v316: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-13T14:27:33.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:33.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:33.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:33.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:33.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:33.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:33.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:33.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:33.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:34.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:34.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:34.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:34.221 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:34.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:34.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:34.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:34.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:34.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:34.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:34.655 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:27:34.655 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:27:34.718 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:34.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:34.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:34.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:34.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:34.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:35.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:35.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:35.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:35.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:35.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:35.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:35.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[103001]: pgmap v317: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1682716684' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[107556]: pgmap v317: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1682716684' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:35 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:35.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:35.504 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:27:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:35 smithi189 ceph-mon[90992]: pgmap v317: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:27:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1682716684' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:35 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:35.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:35.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:35.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:35.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:35.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:35.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:35.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:35.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:36.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:36.119 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:36.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:36.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:36.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:36.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[103001]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:36 smithi017 ceph-mon[107556]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:36.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:36 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:36 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:36 smithi189 ceph-mon[90992]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:36.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:36.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:36.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:36.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:36.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:36.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:36.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:36.967 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:36.998 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:27:36.999 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:27:36.999 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2024-09-13T14:27:37.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:27:37.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:37.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:37.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:37.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:37.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:37.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:37 smithi017 ceph-mon[103001]: pgmap v319: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:27:37.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1966054884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:37.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:37 smithi017 ceph-mon[107556]: pgmap v319: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:27:37.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1966054884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:37.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:37.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:37.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:37 smithi189 ceph-mon[90992]: pgmap v319: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-13T14:27:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1966054884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:37.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:37.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:37.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:37] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:27:37.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:37.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:37.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:37.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:37.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:37.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:27:38.035 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:27:38.036 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:27:38.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:38.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:38.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:38.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:38.263 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:38.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:38.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:38.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:38.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:38.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:38.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:38.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:38.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:38.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:38.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:38.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:38.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:39.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:39.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:39.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:39.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:39.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:39.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:39.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:39 smithi017 ceph-mon[103001]: pgmap v320: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:27:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:39 smithi017 ceph-mon[107556]: pgmap v320: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:27:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:39 smithi189 ceph-mon[90992]: pgmap v320: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:27:39.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:39.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:39.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:27:39.598 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:39.659 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:39.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:39.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:39.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:39.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:27:39.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:39.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:39.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:40.010 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:40.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:40.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:40.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:40.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:40.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:40.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:40.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:40.461 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:40.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:40.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:40.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:40.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:40.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:40.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:40.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:40.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:40.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:40.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:27:41.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:41.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:41.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:41.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:41.385 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:41 smithi189 ceph-mon[90992]: pgmap v321: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-13T14:27:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:41 smithi017 ceph-mon[103001]: pgmap v321: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-13T14:27:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:41 smithi017 ceph-mon[107556]: pgmap v321: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-13T14:27:41.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:41.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:41.619 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:41.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:41.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:41.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:41.805 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:41.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:41.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:41.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:42.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162551 5.5 e116/104 5:a80ac987:::benchmark_data_smithi017_147622_object27922:head v 116'10179, mlcod=116'10177)", 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.060195+0000", 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.016608517999999999, 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016643093000000001, 2024-09-13T14:27:42.079 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060195+0000", 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060195+0000", 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060194+0000", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060201+0000", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2819999999999999e-06 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060209+0000", 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2250000000000003e-06 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060213+0000", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3329999999999998e-06 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060409+0000", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000196178 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.060452+0000", 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2880000000000003e-05 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.082 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162556 5.5 e116/104 5:a40cea94:::benchmark_data_smithi017_147622_object27927:head v 116'10180, mlcod=116'10177)", 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.061702+0000", 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.015100626000000001, 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015200234, 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061702+0000", 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061702+0000", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061702+0000", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061710+0000", 2024-09-13T14:27:42.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5469999999999997e-06 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061725+0000", 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4399999999999999e-05 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.061730+0000", 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.305e-06 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.069496+0000", 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077662809999999999 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.069550+0000", 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4027999999999998e-05 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162557 5.5 e116/104)", 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.066640+0000", 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.010163205999999999, 2024-09-13T14:27:42.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010315693000000001, 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066640+0000", 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066640+0000", 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066639+0000", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066648+0000", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8820000000000007e-06 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066668+0000", 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9854e-05 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.066673+0000", 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6299999999999997e-06 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162559 5.0 e116/104 5:08ed53e3:::benchmark_data_smithi017_147622_object27930:head v 116'10272, mlcod=116'10269)", 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.067291+0000", 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0095120080000000006, 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0097085269999999998, 2024-09-13T14:27:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067291+0000", 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067291+0000", 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067291+0000", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067298+0000", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6279999999999996e-06 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067310+0000", 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1691999999999999e-05 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067314+0000", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5499999999999996e-06 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067352+0000", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7546000000000003e-05 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.067372+0000", 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9527000000000001e-05 2024-09-13T14:27:42.092 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162558 5.a e116/104 5:50e2addd:::benchmark_data_smithi017_147622_object27929:head v 116'10064, mlcod=116'10062)", 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.070268+0000", 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0065354530000000001, 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0067863639999999996, 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070268+0000", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070268+0000", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070267+0000", 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:42.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070277+0000", 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0706e-05 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070292+0000", 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4598e-05 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070297+0000", 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3260000000000003e-06 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070357+0000", 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0044000000000003e-05 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.070378+0000", 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0716999999999998e-05 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162564 5.f e116/104 5:f3260cca:::benchmark_data_smithi017_147622_object27935:head v 116'10043, mlcod=116'10041)", 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.072855+0000", 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0039476320000000004, 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042536559999999998, 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072855+0000", 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072855+0000", 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072855+0000", 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072862+0000", 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7389999999999999e-06 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072874+0000", 2024-09-13T14:27:42.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1849e-05 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072881+0000", 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4420000000000003e-06 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072923+0000", 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2877999999999998e-05 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.072942+0000", 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8474999999999998e-05 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:162565 5.0 e116/104 5:089de972:::benchmark_data_smithi017_147622_object27936:head v 116'10273, mlcod=116'10270)", 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:42.073434+0000", 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.003369169, 2024-09-13T14:27:42.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003730999, 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073434+0000", 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073434+0000", 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:42.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073433+0000", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073444+0000", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.047e-05 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073454+0000", 2024-09-13T14:27:42.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0029e-05 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.073459+0000", 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.075193+0000", 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0017337819999999999 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:42.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:42.075212+0000", 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8808999999999999e-05 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 7 2024-09-13T14:27:42.105 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:27:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:42.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:42.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:42.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:42.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:42.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:42.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:42.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.491 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:42.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:42.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:42.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:42.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:42.686 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:42.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:42.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:42.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:27:42.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:42.921 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:27:42.922 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:27:42.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:42.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:43.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:43.100 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:43.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:43.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:43.321 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:43.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:43.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:43.462 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:43 smithi189 ceph-mon[90992]: pgmap v322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-13T14:27:43.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:43.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:43.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:43.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:43.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:43 smithi017 ceph-mon[103001]: pgmap v322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-13T14:27:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:43 smithi017 ceph-mon[107556]: pgmap v322: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-13T14:27:43.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:43.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:43.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:43.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:43.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:43.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:44.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:44.059 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":322,"stamp":"2024-09-13T14:27:41.773079+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7187666917,"num_objects":109631,"num_object_clones":0,"num_object_copies":328893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109631,"num_whiteouts":0,"num_read":6305,"num_read_kb":13771,"num_write":160707,"num_write_kb":8630316,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147583,"ondisk_log_size":147583,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22797496,"kb_used_data":21263408,"kb_used_omap":35,"kb_used_meta":1533916,"kb_avail":727163720,"statfs":{"total":767960285184,"available":744615649280,"internally_reserved":0,"allocated":21773729792,"data_stored":21768960199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1570731001},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,1,3,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-572719104,"num_objects":-8739,"num_object_clones":0,"num_object_copies":-26217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8739,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002982"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193741+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:27:35.193741+0000","last_peered":"2024-09-13T14:27:35.193741+0000","last_clean":"2024-09-13T14:27:35.193741+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:27:35.193741+0000","last_undegraded":"2024-09-13T14:27:35.193741+0000","last_fullsized":"2024-09-13T14:27:35.193741+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":82,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.138570+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:27:20.138570+0000","last_peered":"2024-09-13T14:27:20.138570+0000","last_clean":"2024-09-13T14:27:20.138570+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:27:20.138570+0000","last_undegraded":"2024-09-13T14:27:20.138570+0000","last_fullsized":"2024-09-13T14:27:20.138570+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133359+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:27:20.133359+0000","last_peered":"2024-09-13T14:27:20.133359+0000","last_clean":"2024-09-13T14:27:20.133359+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:27:20.133359+0000","last_undegraded":"2024-09-13T14:27:20.133359+0000","last_fullsized":"2024-09-13T14:27:20.133359+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.202623+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:27:35.202623+0000","last_peered":"2024-09-13T14:27:35.202623+0000","last_clean":"2024-09-13T14:27:35.202623+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:27:35.202623+0000","last_undegraded":"2024-09-13T14:27:35.202623+0000","last_fullsized":"2024-09-13T14:27:35.202623+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":94,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203445+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:27:35.203445+0000","last_peered":"2024-09-13T14:27:35.203445+0000","last_clean":"2024-09-13T14:27:35.203445+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:27:35.203445+0000","last_undegraded":"2024-09-13T14:27:35.203445+0000","last_fullsized":"2024-09-13T14:27:35.203445+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":63,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.144860+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:27:20.144860+0000","last_peered":"2024-09-13T14:27:20.144860+0000","last_clean":"2024-09-13T14:27:20.144860+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:27:20.144860+0000","last_undegraded":"2024-09-13T14:27:20.144860+0000","last_fullsized":"2024-09-13T14:27:20.144860+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.194708+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:27:35.194708+0000","last_peered":"2024-09-13T14:27:35.194708+0000","last_clean":"2024-09-13T14:27:35.194708+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:27:35.194708+0000","last_undegraded":"2024-09-13T14:27:35.194708+0000","last_fullsized":"2024-09-13T14:27:35.194708+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":88,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.122750+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:27:20.122750+0000","last_peered":"2024-09-13T14:27:20.122750+0000","last_clean":"2024-09-13T14:27:20.122750+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:27:20.122750+0000","last_undegraded":"2024-09-13T14:27:20.122750+0000","last_fullsized":"2024-09-13T14:27:20.122750+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"116'9892","reported_seq":9912,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.748469+0000","last_change":"2024-09-13T14:25:29.103229+0000","last_active":"2024-09-13T14:27:37.748469+0000","last_peered":"2024-09-13T14:27:37.748469+0000","last_clean":"2024-09-13T14:27:37.748469+0000","last_became_active":"2024-09-13T14:25:29.102723+0000","last_became_peered":"2024-09-13T14:25:29.102723+0000","last_unstale":"2024-09-13T14:27:37.748469+0000","last_undegraded":"2024-09-13T14:27:37.748469+0000","last_fullsized":"2024-09-13T14:27:37.748469+0000","mapping_epoch":104,"log_start":"108'1300","ondisk_log_start":"108'1300","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8592,"log_dups_size":0,"ondisk_log_size":8592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:22.830419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452329472,"num_objects":6902,"num_object_clones":0,"num_object_copies":20706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9892,"num_write_kb":537408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":81,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.138979+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:27:20.138979+0000","last_peered":"2024-09-13T14:27:20.138979+0000","last_clean":"2024-09-13T14:27:20.138979+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:27:20.138979+0000","last_undegraded":"2024-09-13T14:27:20.138979+0000","last_fullsized":"2024-09-13T14:27:20.138979+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":62,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133376+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:27:20.133376+0000","last_peered":"2024-09-13T14:27:20.133376+0000","last_clean":"2024-09-13T14:27:20.133376+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:27:20.133376+0000","last_undegraded":"2024-09-13T14:27:20.133376+0000","last_fullsized":"2024-09-13T14:27:20.133376+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"116'10076","reported_seq":10112,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:40.645937+0000","last_change":"2024-09-13T14:25:29.106872+0000","last_active":"2024-09-13T14:27:40.645937+0000","last_peered":"2024-09-13T14:27:40.645937+0000","last_clean":"2024-09-13T14:27:40.645937+0000","last_became_active":"2024-09-13T14:25:29.106582+0000","last_became_peered":"2024-09-13T14:25:29.106582+0000","last_unstale":"2024-09-13T14:27:40.645937+0000","last_undegraded":"2024-09-13T14:27:40.645937+0000","last_fullsized":"2024-09-13T14:27:40.645937+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:29.316375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441319424,"num_objects":6734,"num_object_clones":0,"num_object_copies":20202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10076,"num_write_kb":537920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":82,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139323+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:27:20.139323+0000","last_peered":"2024-09-13T14:27:20.139323+0000","last_clean":"2024-09-13T14:27:20.139323+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:27:20.139323+0000","last_undegraded":"2024-09-13T14:27:20.139323+0000","last_fullsized":"2024-09-13T14:27:20.139323+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139684+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:27:20.139684+0000","last_peered":"2024-09-13T14:27:20.139684+0000","last_clean":"2024-09-13T14:27:20.139684+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:27:20.139684+0000","last_undegraded":"2024-09-13T14:27:20.139684+0000","last_fullsized":"2024-09-13T14:27:20.139684+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"116'10312","reported_seq":10347,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:40.627429+0000","last_change":"2024-09-13T14:25:29.106992+0000","last_active":"2024-09-13T14:27:40.627429+0000","last_peered":"2024-09-13T14:27:40.627429+0000","last_clean":"2024-09-13T14:27:40.627429+0000","last_became_active":"2024-09-13T14:25:29.106698+0000","last_became_peered":"2024-09-13T14:25:29.106698+0000","last_unstale":"2024-09-13T14:27:40.627429+0000","last_undegraded":"2024-09-13T14:27:40.627429+0000","last_fullsized":"2024-09-13T14:27:40.627429+0000","mapping_epoch":104,"log_start":"108'300","ondisk_log_start":"108'300","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10012,"log_dups_size":0,"ondisk_log_size":10012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:20.608913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448921600,"num_objects":6850,"num_object_clones":0,"num_object_copies":20550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10312,"num_write_kb":549184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":64,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.143735+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:27:20.143735+0000","last_peered":"2024-09-13T14:27:20.143735+0000","last_clean":"2024-09-13T14:27:20.143735+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:27:20.143735+0000","last_undegraded":"2024-09-13T14:27:20.143735+0000","last_fullsized":"2024-09-13T14:27:20.143735+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.125890+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:27:20.125890+0000","last_peered":"2024-09-13T14:27:20.125890+0000","last_clean":"2024-09-13T14:27:20.125890+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:27:20.125890+0000","last_undegraded":"2024-09-13T14:27:20.125890+0000","last_fullsized":"2024-09-13T14:27:20.125890+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"116'10008","reported_seq":10043,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:40.633223+0000","last_change":"2024-09-13T14:25:29.103217+0000","last_active":"2024-09-13T14:27:40.633223+0000","last_peered":"2024-09-13T14:27:40.633223+0000","last_clean":"2024-09-13T14:27:40.633223+0000","last_became_active":"2024-09-13T14:25:29.102866+0000","last_became_peered":"2024-09-13T14:25:29.102866+0000","last_unstale":"2024-09-13T14:27:40.633223+0000","last_undegraded":"2024-09-13T14:27:40.633223+0000","last_fullsized":"2024-09-13T14:27:40.633223+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:50.615492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433455104,"num_objects":6614,"num_object_clones":0,"num_object_copies":19842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10008,"num_write_kb":531904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133108+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:27:20.133108+0000","last_peered":"2024-09-13T14:27:20.133108+0000","last_clean":"2024-09-13T14:27:20.133108+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:27:20.133108+0000","last_undegraded":"2024-09-13T14:27:20.133108+0000","last_fullsized":"2024-09-13T14:27:20.133108+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":94,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203002+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:27:35.203002+0000","last_peered":"2024-09-13T14:27:35.203002+0000","last_clean":"2024-09-13T14:27:35.203002+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:27:35.203002+0000","last_undegraded":"2024-09-13T14:27:35.203002+0000","last_fullsized":"2024-09-13T14:27:35.203002+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"116'10207","reported_seq":10242,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:40.620413+0000","last_change":"2024-09-13T14:25:29.106845+0000","last_active":"2024-09-13T14:27:40.620413+0000","last_peered":"2024-09-13T14:27:40.620413+0000","last_clean":"2024-09-13T14:27:40.620413+0000","last_became_active":"2024-09-13T14:25:29.106506+0000","last_became_peered":"2024-09-13T14:25:29.106506+0000","last_unstale":"2024-09-13T14:27:40.620413+0000","last_undegraded":"2024-09-13T14:27:40.620413+0000","last_fullsized":"2024-09-13T14:27:40.620413+0000","mapping_epoch":104,"log_start":"108'200","ondisk_log_start":"108'200","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:30:38.859386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446300184,"num_objects":6811,"num_object_clones":0,"num_object_copies":20433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6811,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10207,"num_write_kb":544513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":95,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193564+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:27:35.193564+0000","last_peered":"2024-09-13T14:27:35.193564+0000","last_clean":"2024-09-13T14:27:35.193564+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:27:35.193564+0000","last_undegraded":"2024-09-13T14:27:35.193564+0000","last_fullsized":"2024-09-13T14:27:35.193564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136542+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:27:20.136542+0000","last_peered":"2024-09-13T14:27:20.136542+0000","last_clean":"2024-09-13T14:27:20.136542+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:27:20.136542+0000","last_undegraded":"2024-09-13T14:27:20.136542+0000","last_fullsized":"2024-09-13T14:27:20.136542+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"116'9761","reported_seq":9796,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:36.953440+0000","last_change":"2024-09-13T14:25:29.103072+0000","last_active":"2024-09-13T14:27:36.953440+0000","last_peered":"2024-09-13T14:27:36.953440+0000","last_clean":"2024-09-13T14:27:36.953440+0000","last_became_active":"2024-09-13T14:25:29.102737+0000","last_became_peered":"2024-09-13T14:25:29.102737+0000","last_unstale":"2024-09-13T14:27:36.953440+0000","last_undegraded":"2024-09-13T14:27:36.953440+0000","last_fullsized":"2024-09-13T14:27:36.953440+0000","mapping_epoch":104,"log_start":"109'1600","ondisk_log_start":"109'1600","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:42.212762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":454754304,"num_objects":6939,"num_object_clones":0,"num_object_copies":20817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9761,"num_write_kb":534400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.123063+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:27:20.123063+0000","last_peered":"2024-09-13T14:27:20.123063+0000","last_clean":"2024-09-13T14:27:20.123063+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:27:20.123063+0000","last_undegraded":"2024-09-13T14:27:20.123063+0000","last_fullsized":"2024-09-13T14:27:20.123063+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139768+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:27:20.139768+0000","last_peered":"2024-09-13T14:27:20.139768+0000","last_clean":"2024-09-13T14:27:20.139768+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:27:20.139768+0000","last_undegraded":"2024-09-13T14:27:20.139768+0000","last_fullsized":"2024-09-13T14:27:20.139768+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"116'9808","reported_seq":9828,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.813108+0000","last_change":"2024-09-13T14:25:29.105948+0000","last_active":"2024-09-13T14:27:37.813108+0000","last_peered":"2024-09-13T14:27:37.813108+0000","last_clean":"2024-09-13T14:27:37.813108+0000","last_became_active":"2024-09-13T14:25:29.105674+0000","last_became_peered":"2024-09-13T14:25:29.105674+0000","last_unstale":"2024-09-13T14:27:37.813108+0000","last_undegraded":"2024-09-13T14:27:37.813108+0000","last_fullsized":"2024-09-13T14:27:37.813108+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":9808,"log_dups_size":0,"ondisk_log_size":9808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:11:26.842644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448790528,"num_objects":6848,"num_object_clones":0,"num_object_copies":20544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9808,"num_write_kb":532992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":81,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139746+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:27:20.139746+0000","last_peered":"2024-09-13T14:27:20.139746+0000","last_clean":"2024-09-13T14:27:20.139746+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:27:20.139746+0000","last_undegraded":"2024-09-13T14:27:20.139746+0000","last_fullsized":"2024-09-13T14:27:20.139746+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.137085+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:27:20.137085+0000","last_peered":"2024-09-13T14:27:20.137085+0000","last_clean":"2024-09-13T14:27:20.137085+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:27:20.137085+0000","last_undegraded":"2024-09-13T14:27:20.137085+0000","last_fullsized":"2024-09-13T14:27:20.137085+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"116'9830","reported_seq":9850,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.790465+0000","last_change":"2024-09-13T14:25:29.105317+0000","last_active":"2024-09-13T14:27:37.790465+0000","last_peered":"2024-09-13T14:27:37.790465+0000","last_clean":"2024-09-13T14:27:37.790465+0000","last_became_active":"2024-09-13T14:25:29.104986+0000","last_became_peered":"2024-09-13T14:25:29.104986+0000","last_unstale":"2024-09-13T14:27:37.790465+0000","last_undegraded":"2024-09-13T14:27:37.790465+0000","last_fullsized":"2024-09-13T14:27:37.790465+0000","mapping_epoch":104,"log_start":"109'2500","ondisk_log_start":"109'2500","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:21.431572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446038016,"num_objects":6806,"num_object_clones":0,"num_object_copies":20418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9830,"num_write_kb":532352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193481+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:27:35.193481+0000","last_peered":"2024-09-13T14:27:35.193481+0000","last_clean":"2024-09-13T14:27:35.193481+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:27:35.193481+0000","last_undegraded":"2024-09-13T14:27:35.193481+0000","last_fullsized":"2024-09-13T14:27:35.193481+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.200904+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:27:35.200904+0000","last_peered":"2024-09-13T14:27:35.200904+0000","last_clean":"2024-09-13T14:27:35.200904+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:27:35.200904+0000","last_undegraded":"2024-09-13T14:27:35.200904+0000","last_fullsized":"2024-09-13T14:27:35.200904+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"116'9905","reported_seq":9925,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.751077+0000","last_change":"2024-09-13T14:25:29.104481+0000","last_active":"2024-09-13T14:27:37.751077+0000","last_peered":"2024-09-13T14:27:37.751077+0000","last_clean":"2024-09-13T14:27:37.751077+0000","last_became_active":"2024-09-13T14:25:29.104179+0000","last_became_peered":"2024-09-13T14:25:29.104179+0000","last_unstale":"2024-09-13T14:27:37.751077+0000","last_undegraded":"2024-09-13T14:27:37.751077+0000","last_fullsized":"2024-09-13T14:27:37.751077+0000","mapping_epoch":104,"log_start":"108'1300","ondisk_log_start":"108'1300","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:33:03.512833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448200704,"num_objects":6839,"num_object_clones":0,"num_object_copies":20517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9905,"num_write_kb":535808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":197,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.138491+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:27:20.138491+0000","last_peered":"2024-09-13T14:27:20.138491+0000","last_clean":"2024-09-13T14:27:20.138491+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:27:20.138491+0000","last_undegraded":"2024-09-13T14:27:20.138491+0000","last_fullsized":"2024-09-13T14:27:20.138491+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":681,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.135659+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:27:20.135659+0000","last_peered":"2024-09-13T14:27:20.135659+0000","last_clean":"2024-09-13T14:27:20.135659+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:27:20.135659+0000","last_undegraded":"2024-09-13T14:27:20.135659+0000","last_fullsized":"2024-09-13T14:27:20.135659+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.197151+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:27:35.197151+0000","last_peered":"2024-09-13T14:27:35.197151+0000","last_clean":"2024-09-13T14:27:35.197151+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:27:35.197151+0000","last_undegraded":"2024-09-13T14:27:35.197151+0000","last_fullsized":"2024-09-13T14:27:35.197151+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.126322+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:27:20.126322+0000","last_peered":"2024-09-13T14:27:20.126322+0000","last_clean":"2024-09-13T14:27:20.126322+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:27:20.126322+0000","last_undegraded":"2024-09-13T14:27:20.126322+0000","last_fullsized":"2024-09-13T14:27:20.126322+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"116'10001","reported_seq":10021,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.838134+0000","last_change":"2024-09-13T14:25:29.105952+0000","last_active":"2024-09-13T14:27:37.838134+0000","last_peered":"2024-09-13T14:27:37.838134+0000","last_clean":"2024-09-13T14:27:37.838134+0000","last_became_active":"2024-09-13T14:25:29.105063+0000","last_became_peered":"2024-09-13T14:25:29.105063+0000","last_unstale":"2024-09-13T14:27:37.838134+0000","last_undegraded":"2024-09-13T14:27:37.838134+0000","last_fullsized":"2024-09-13T14:27:37.838134+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:05.015793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":455016448,"num_objects":6943,"num_object_clones":0,"num_object_copies":20829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10001,"num_write_kb":542208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":489,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.138411+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:27:20.138411+0000","last_peered":"2024-09-13T14:27:20.138411+0000","last_clean":"2024-09-13T14:27:20.138411+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:27:20.138411+0000","last_undegraded":"2024-09-13T14:27:20.138411+0000","last_fullsized":"2024-09-13T14:27:20.138411+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":93,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.202434+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:27:35.202434+0000","last_peered":"2024-09-13T14:27:35.202434+0000","last_clean":"2024-09-13T14:27:35.202434+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:27:35.202434+0000","last_undegraded":"2024-09-13T14:27:35.202434+0000","last_fullsized":"2024-09-13T14:27:35.202434+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":105,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203645+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:27:35.203645+0000","last_peered":"2024-09-13T14:27:35.203645+0000","last_clean":"2024-09-13T14:27:35.203645+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:27:35.203645+0000","last_undegraded":"2024-09-13T14:27:35.203645+0000","last_fullsized":"2024-09-13T14:27:35.203645+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"116'10067","reported_seq":10103,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:40.589523+0000","last_change":"2024-09-13T14:25:29.106186+0000","last_active":"2024-09-13T14:27:40.589523+0000","last_peered":"2024-09-13T14:27:40.589523+0000","last_clean":"2024-09-13T14:27:40.589523+0000","last_became_active":"2024-09-13T14:25:29.105860+0000","last_became_peered":"2024-09-13T14:25:29.105860+0000","last_unstale":"2024-09-13T14:27:40.589523+0000","last_undegraded":"2024-09-13T14:27:40.589523+0000","last_fullsized":"2024-09-13T14:27:40.589523+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10067,"log_dups_size":0,"ondisk_log_size":10067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:06.384244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440598528,"num_objects":6723,"num_object_clones":0,"num_object_copies":20169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10067,"num_write_kb":537280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":523,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.188598+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:27:35.188598+0000","last_peered":"2024-09-13T14:27:35.188598+0000","last_clean":"2024-09-13T14:27:35.188598+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:27:35.188598+0000","last_undegraded":"2024-09-13T14:27:35.188598+0000","last_fullsized":"2024-09-13T14:27:35.188598+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.202795+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:27:35.202795+0000","last_peered":"2024-09-13T14:27:35.202795+0000","last_clean":"2024-09-13T14:27:35.202795+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:27:35.202795+0000","last_undegraded":"2024-09-13T14:27:35.202795+0000","last_fullsized":"2024-09-13T14:27:35.202795+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":99,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.187655+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:27:35.187655+0000","last_peered":"2024-09-13T14:27:35.187655+0000","last_clean":"2024-09-13T14:27:35.187655+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:27:35.187655+0000","last_undegraded":"2024-09-13T14:27:35.187655+0000","last_fullsized":"2024-09-13T14:27:35.187655+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.5","version":"116'9913","reported_seq":9949,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.201810+0000","last_change":"2024-09-13T14:25:29.105222+0000","last_active":"2024-09-13T14:27:37.201810+0000","last_peered":"2024-09-13T14:27:37.201810+0000","last_clean":"2024-09-13T14:27:37.201810+0000","last_became_active":"2024-09-13T14:25:29.104759+0000","last_became_peered":"2024-09-13T14:25:29.104759+0000","last_unstale":"2024-09-13T14:27:37.201810+0000","last_undegraded":"2024-09-13T14:27:37.201810+0000","last_fullsized":"2024-09-13T14:27:37.201810+0000","mapping_epoch":104,"log_start":"108'1000","ondisk_log_start":"108'1000","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8913,"log_dups_size":0,"ondisk_log_size":8913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:25.238466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458555392,"num_objects":6997,"num_object_clones":0,"num_object_copies":20991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9913,"num_write_kb":541120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":730,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.132993+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:27:20.132993+0000","last_peered":"2024-09-13T14:27:20.132993+0000","last_clean":"2024-09-13T14:27:20.132993+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:27:20.132993+0000","last_undegraded":"2024-09-13T14:27:20.132993+0000","last_fullsized":"2024-09-13T14:27:20.132993+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136376+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:27:20.136376+0000","last_peered":"2024-09-13T14:27:20.136376+0000","last_clean":"2024-09-13T14:27:20.136376+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:27:20.136376+0000","last_undegraded":"2024-09-13T14:27:20.136376+0000","last_fullsized":"2024-09-13T14:27:20.136376+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.200860+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:27:35.200860+0000","last_peered":"2024-09-13T14:27:35.200860+0000","last_clean":"2024-09-13T14:27:35.200860+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:27:35.200860+0000","last_undegraded":"2024-09-13T14:27:35.200860+0000","last_fullsized":"2024-09-13T14:27:35.200860+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"116'9919","reported_seq":9948,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.194949+0000","last_change":"2024-09-13T14:25:29.104562+0000","last_active":"2024-09-13T14:27:37.194949+0000","last_peered":"2024-09-13T14:27:37.194949+0000","last_clean":"2024-09-13T14:27:37.194949+0000","last_became_active":"2024-09-13T14:25:29.104246+0000","last_became_peered":"2024-09-13T14:25:29.104246+0000","last_unstale":"2024-09-13T14:27:37.194949+0000","last_undegraded":"2024-09-13T14:27:37.194949+0000","last_fullsized":"2024-09-13T14:27:37.194949+0000","mapping_epoch":104,"log_start":"108'1000","ondisk_log_start":"108'1000","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8919,"log_dups_size":0,"ondisk_log_size":8919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:48.350655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":457244672,"num_objects":6977,"num_object_clones":0,"num_object_copies":20931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9919,"num_write_kb":540672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133588+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:27:20.133588+0000","last_peered":"2024-09-13T14:27:20.133588+0000","last_clean":"2024-09-13T14:27:20.133588+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:27:20.133588+0000","last_undegraded":"2024-09-13T14:27:20.133588+0000","last_fullsized":"2024-09-13T14:27:20.133588+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":694,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.201047+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:27:35.201047+0000","last_peered":"2024-09-13T14:27:35.201047+0000","last_clean":"2024-09-13T14:27:35.201047+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:27:35.201047+0000","last_undegraded":"2024-09-13T14:27:35.201047+0000","last_fullsized":"2024-09-13T14:27:35.201047+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.199214+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:27:35.199214+0000","last_peered":"2024-09-13T14:27:35.199214+0000","last_clean":"2024-09-13T14:27:35.199214+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:27:35.199214+0000","last_undegraded":"2024-09-13T14:27:35.199214+0000","last_fullsized":"2024-09-13T14:27:35.199214+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"116'9920","reported_seq":9938,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.739701+0000","last_change":"2024-09-13T14:25:29.104951+0000","last_active":"2024-09-13T14:27:37.739701+0000","last_peered":"2024-09-13T14:27:37.739701+0000","last_clean":"2024-09-13T14:27:37.739701+0000","last_became_active":"2024-09-13T14:25:29.104323+0000","last_became_peered":"2024-09-13T14:25:29.104323+0000","last_unstale":"2024-09-13T14:27:37.739701+0000","last_undegraded":"2024-09-13T14:27:37.739701+0000","last_fullsized":"2024-09-13T14:27:37.739701+0000","mapping_epoch":104,"log_start":"108'200","ondisk_log_start":"108'200","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":9720,"log_dups_size":0,"ondisk_log_size":9720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:22.616365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452984832,"num_objects":6912,"num_object_clones":0,"num_object_copies":20736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9920,"num_write_kb":538624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.200956+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:27:35.200956+0000","last_peered":"2024-09-13T14:27:35.200956+0000","last_clean":"2024-09-13T14:27:35.200956+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:27:35.200956+0000","last_undegraded":"2024-09-13T14:27:35.200956+0000","last_fullsized":"2024-09-13T14:27:35.200956+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":426,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.144011+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:27:20.144011+0000","last_peered":"2024-09-13T14:27:20.144011+0000","last_clean":"2024-09-13T14:27:20.144011+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:27:20.144011+0000","last_undegraded":"2024-09-13T14:27:20.144011+0000","last_fullsized":"2024-09-13T14:27:20.144011+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":83,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.202949+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:27:35.202949+0000","last_peered":"2024-09-13T14:27:35.202949+0000","last_clean":"2024-09-13T14:27:35.202949+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:27:35.202949+0000","last_undegraded":"2024-09-13T14:27:35.202949+0000","last_fullsized":"2024-09-13T14:27:35.202949+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"116'9936","reported_seq":9956,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:37.788897+0000","last_change":"2024-09-13T14:25:29.103444+0000","last_active":"2024-09-13T14:27:37.788897+0000","last_peered":"2024-09-13T14:27:37.788897+0000","last_clean":"2024-09-13T14:27:37.788897+0000","last_became_active":"2024-09-13T14:25:29.102571+0000","last_became_peered":"2024-09-13T14:25:29.102571+0000","last_unstale":"2024-09-13T14:27:37.788897+0000","last_undegraded":"2024-09-13T14:27:37.788897+0000","last_fullsized":"2024-09-13T14:27:37.788897+0000","mapping_epoch":104,"log_start":"109'2600","ondisk_log_start":"109'2600","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":7336,"log_dups_size":0,"ondisk_log_size":7336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:32.020793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447873024,"num_objects":6834,"num_object_clones":0,"num_object_copies":20502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9936,"num_write_kb":536640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":95,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.194893+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:27:35.194893+0000","last_peered":"2024-09-13T14:27:35.194893+0000","last_clean":"2024-09-13T14:27:35.194893+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:27:35.194893+0000","last_undegraded":"2024-09-13T14:27:35.194893+0000","last_fullsized":"2024-09-13T14:27:35.194893+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":428,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.199740+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:27:35.199740+0000","last_peered":"2024-09-13T14:27:35.199740+0000","last_clean":"2024-09-13T14:27:35.199740+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:27:35.199740+0000","last_undegraded":"2024-09-13T14:27:35.199740+0000","last_fullsized":"2024-09-13T14:27:35.199740+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193294+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:27:35.193294+0000","last_peered":"2024-09-13T14:27:35.193294+0000","last_clean":"2024-09-13T14:27:35.193294+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:27:35.193294+0000","last_undegraded":"2024-09-13T14:27:35.193294+0000","last_fullsized":"2024-09-13T14:27:35.193294+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"116'10096","reported_seq":10132,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:38.713209+0000","last_change":"2024-09-13T14:25:29.103771+0000","last_active":"2024-09-13T14:27:38.713209+0000","last_peered":"2024-09-13T14:27:38.713209+0000","last_clean":"2024-09-13T14:27:38.713209+0000","last_became_active":"2024-09-13T14:25:29.103394+0000","last_became_peered":"2024-09-13T14:25:29.103394+0000","last_unstale":"2024-09-13T14:27:38.713209+0000","last_undegraded":"2024-09-13T14:27:38.713209+0000","last_fullsized":"2024-09-13T14:27:38.713209+0000","mapping_epoch":104,"log_start":"108'1200","ondisk_log_start":"108'1200","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":8896,"log_dups_size":0,"ondisk_log_size":8896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:12.793430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450494464,"num_objects":6874,"num_object_clones":0,"num_object_copies":20622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10096,"num_write_kb":543040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203352+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:27:35.203352+0000","last_peered":"2024-09-13T14:27:35.203352+0000","last_clean":"2024-09-13T14:27:35.203352+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:27:35.203352+0000","last_undegraded":"2024-09-13T14:27:35.203352+0000","last_fullsized":"2024-09-13T14:27:35.203352+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":534,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.126312+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:27:20.126312+0000","last_peered":"2024-09-13T14:27:20.126312+0000","last_clean":"2024-09-13T14:27:20.126312+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:27:20.126312+0000","last_undegraded":"2024-09-13T14:27:20.126312+0000","last_fullsized":"2024-09-13T14:27:20.126312+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":63,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139574+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:27:20.139574+0000","last_peered":"2024-09-13T14:27:20.139574+0000","last_clean":"2024-09-13T14:27:20.139574+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:27:20.139574+0000","last_undegraded":"2024-09-13T14:27:20.139574+0000","last_fullsized":"2024-09-13T14:27:20.139574+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136262+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:27:20.136262+0000","last_peered":"2024-09-13T14:27:20.136262+0000","last_clean":"2024-09-13T14:27:20.136262+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:27:20.136262+0000","last_undegraded":"2024-09-13T14:27:20.136262+0000","last_fullsized":"2024-09-13T14:27:20.136262+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":86,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136610+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:27:20.136610+0000","last_peered":"2024-09-13T14:27:20.136610+0000","last_clean":"2024-09-13T14:27:20.136610+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:27:20.136610+0000","last_undegraded":"2024-09-13T14:27:20.136610+0000","last_fullsized":"2024-09-13T14:27:20.136610+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.199076+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:27:35.199076+0000","last_peered":"2024-09-13T14:27:35.199076+0000","last_clean":"2024-09-13T14:27:35.199076+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:27:35.199076+0000","last_undegraded":"2024-09-13T14:27:35.199076+0000","last_fullsized":"2024-09-13T14:27:35.199076+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":83,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.143212+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:27:20.143212+0000","last_peered":"2024-09-13T14:27:20.143212+0000","last_clean":"2024-09-13T14:27:20.143212+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:27:20.143212+0000","last_undegraded":"2024-09-13T14:27:20.143212+0000","last_fullsized":"2024-09-13T14:27:20.143212+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":80,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.134707+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:27:20.134707+0000","last_peered":"2024-09-13T14:27:20.134707+0000","last_clean":"2024-09-13T14:27:20.134707+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:27:20.134707+0000","last_undegraded":"2024-09-13T14:27:20.134707+0000","last_fullsized":"2024-09-13T14:27:20.134707+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":52,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193312+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:27:35.193312+0000","last_peered":"2024-09-13T14:27:35.193312+0000","last_clean":"2024-09-13T14:27:35.193312+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:27:35.193312+0000","last_undegraded":"2024-09-13T14:27:35.193312+0000","last_fullsized":"2024-09-13T14:27:35.193312+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":93,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203228+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:27:35.203228+0000","last_peered":"2024-09-13T14:27:35.203228+0000","last_clean":"2024-09-13T14:27:35.203228+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:27:35.203228+0000","last_undegraded":"2024-09-13T14:27:35.203228+0000","last_fullsized":"2024-09-13T14:27:35.203228+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193197+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:27:35.193197+0000","last_peered":"2024-09-13T14:27:35.193197+0000","last_clean":"2024-09-13T14:27:35.193197+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:27:35.193197+0000","last_undegraded":"2024-09-13T14:27:35.193197+0000","last_fullsized":"2024-09-13T14:27:35.193197+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.126234+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:27:20.126234+0000","last_peered":"2024-09-13T14:27:20.126234+0000","last_clean":"2024-09-13T14:27:20.126234+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:27:20.126234+0000","last_undegraded":"2024-09-13T14:27:20.126234+0000","last_fullsized":"2024-09-13T14:27:20.126234+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":115,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193384+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:27:35.193384+0000","last_peered":"2024-09-13T14:27:35.193384+0000","last_clean":"2024-09-13T14:27:35.193384+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:27:35.193384+0000","last_undegraded":"2024-09-13T14:27:35.193384+0000","last_fullsized":"2024-09-13T14:27:35.193384+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203600+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:27:35.203600+0000","last_peered":"2024-09-13T14:27:35.203600+0000","last_clean":"2024-09-13T14:27:35.203600+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:27:35.203600+0000","last_undegraded":"2024-09-13T14:27:35.203600+0000","last_fullsized":"2024-09-13T14:27:35.203600+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.198847+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:27:35.198847+0000","last_peered":"2024-09-13T14:27:35.198847+0000","last_clean":"2024-09-13T14:27:35.198847+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:27:35.198847+0000","last_undegraded":"2024-09-13T14:27:35.198847+0000","last_fullsized":"2024-09-13T14:27:35.198847+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.194005+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:27:35.194005+0000","last_peered":"2024-09-13T14:27:35.194005+0000","last_clean":"2024-09-13T14:27:35.194005+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:27:35.194005+0000","last_undegraded":"2024-09-13T14:27:35.194005+0000","last_fullsized":"2024-09-13T14:27:35.194005+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":99,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136155+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:27:20.136155+0000","last_peered":"2024-09-13T14:27:20.136155+0000","last_clean":"2024-09-13T14:27:20.136155+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:27:20.136155+0000","last_undegraded":"2024-09-13T14:27:20.136155+0000","last_fullsized":"2024-09-13T14:27:20.136155+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.188077+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:27:35.188077+0000","last_peered":"2024-09-13T14:27:35.188077+0000","last_clean":"2024-09-13T14:27:35.188077+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:27:35.188077+0000","last_undegraded":"2024-09-13T14:27:35.188077+0000","last_fullsized":"2024-09-13T14:27:35.188077+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":86,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.202910+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:27:35.202910+0000","last_peered":"2024-09-13T14:27:35.202910+0000","last_clean":"2024-09-13T14:27:35.202910+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:27:35.202910+0000","last_undegraded":"2024-09-13T14:27:35.202910+0000","last_fullsized":"2024-09-13T14:27:35.202910+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.139888+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:27:20.139888+0000","last_peered":"2024-09-13T14:27:20.139888+0000","last_clean":"2024-09-13T14:27:20.139888+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:27:20.139888+0000","last_undegraded":"2024-09-13T14:27:20.139888+0000","last_fullsized":"2024-09-13T14:27:20.139888+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":82,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133032+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:27:20.133032+0000","last_peered":"2024-09-13T14:27:20.133032+0000","last_clean":"2024-09-13T14:27:20.133032+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:27:20.133032+0000","last_undegraded":"2024-09-13T14:27:20.133032+0000","last_fullsized":"2024-09-13T14:27:20.133032+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":97,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.136072+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:27:20.136072+0000","last_peered":"2024-09-13T14:27:20.136072+0000","last_clean":"2024-09-13T14:27:20.136072+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:27:20.136072+0000","last_undegraded":"2024-09-13T14:27:20.136072+0000","last_fullsized":"2024-09-13T14:27:20.136072+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":65,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.133513+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:27:20.133513+0000","last_peered":"2024-09-13T14:27:20.133513+0000","last_clean":"2024-09-13T14:27:20.133513+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:27:20.133513+0000","last_undegraded":"2024-09-13T14:27:20.133513+0000","last_fullsized":"2024-09-13T14:27:20.133513+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":118,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.193681+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:27:35.193681+0000","last_peered":"2024-09-13T14:27:35.193681+0000","last_clean":"2024-09-13T14:27:35.193681+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:27:35.193681+0000","last_undegraded":"2024-09-13T14:27:35.193681+0000","last_fullsized":"2024-09-13T14:27:35.193681+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":103,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.194599+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:27:35.194599+0000","last_peered":"2024-09-13T14:27:35.194599+0000","last_clean":"2024-09-13T14:27:35.194599+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:27:35.194599+0000","last_undegraded":"2024-09-13T14:27:35.194599+0000","last_fullsized":"2024-09-13T14:27:35.194599+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":90,"reported_epoch":115,"state":"active+clean","last_fresh":"2024-09-13T14:27:20.135985+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:27:20.135985+0000","last_peered":"2024-09-13T14:27:20.135985+0000","last_clean":"2024-09-13T14:27:20.135985+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:27:20.135985+0000","last_undegraded":"2024-09-13T14:27:20.135985+0000","last_fullsized":"2024-09-13T14:27:20.135985+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-09-13T14:27:35.203136+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:27:35.203136+0000","last_peered":"2024-09-13T14:27:35.203136+0000","last_clean":"2024-09-13T14:27:35.203136+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:27:35.203136+0000","last_undegraded":"2024-09-13T14:27:35.203136+0000","last_fullsized":"2024-09-13T14:27:35.203136+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":7182876696,"num_objects":109603,"num_object_clones":0,"num_object_copies":328809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109603,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159651,"num_write_kb":8616065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21650812928,"data_stored":21650800712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146451,"ondisk_log_size":146451,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397698,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527508,"kb_used_data":2225996,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91217644,"statfs":{"total":95995035648,"available":93406867456,"internally_reserved":0,"allocated":2279419904,"data_stored":2278804593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593926,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2082560,"kb_used_data":1781048,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91662592,"statfs":{"total":95995035648,"available":93862494208,"internally_reserved":0,"allocated":1823793152,"data_stored":1823197603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790156,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4298424,"kb_used_data":3992688,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":89446728,"statfs":{"total":95995035648,"available":91593449472,"internally_reserved":0,"allocated":4088512512,"data_stored":4087920398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986384,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3852940,"kb_used_data":3538944,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":89892212,"statfs":{"total":95995035648,"available":92049625088,"internally_reserved":0,"allocated":3623878656,"data_stored":3623274995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696942,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357940,"kb_used_data":1312428,"kb_used_omap":4,"kb_used_meta":45499,"kb_avail":92387212,"statfs":{"total":95995035648,"available":94604505088,"internally_reserved":0,"allocated":1343926272,"data_stored":1343338303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":46591051},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860469,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2688380,"kb_used_data":2602548,"kb_used_omap":8,"kb_used_meta":85815,"kb_avail":91056772,"statfs":{"total":95995035648,"available":93242134528,"internally_reserved":0,"allocated":2665009152,"data_stored":2664413537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":87875404},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":91,"seq":390842023995,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2304824,"kb_used_data":2237036,"kb_used_omap":7,"kb_used_meta":67768,"kb_avail":91440328,"statfs":{"total":95995035648,"available":93634895872,"internally_reserved":0,"allocated":2290724864,"data_stored":2290136816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":69394871},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187530,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3684920,"kb_used_data":3572720,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":90060232,"statfs":{"total":95995035648,"available":92221677568,"internally_reserved":0,"allocated":3658465280,"data_stored":3657873954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":114876747},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3645046784,"data_stored":3645046784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2282946560,"data_stored":2282946560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2647330816,"data_stored":2647326744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1335427072,"data_stored":1335427072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3601530880,"data_stored":3601530880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4072280064,"data_stored":4072275992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1809448960,"data_stored":1809448960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2256801792,"data_stored":2256797720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:27:44.063 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:27:44.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_0 pg_num 26 2024-09-13T14:27:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:44.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:44.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:44.343 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:44 smithi017 ceph-mon[103001]: from='client.45047 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:44.343 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:44 smithi017 ceph-mon[107556]: from='client.45047 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:44.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:44 smithi189 ceph-mon[90992]: from='client.45047 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:27:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:44.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:44.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:44.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:44.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:44.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:44.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:44.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:45.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:45.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:45.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:45.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:45.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:45.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:45.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:45.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:45 smithi189 ceph-mon[90992]: pgmap v323: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:27:45.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:45 smithi017 ceph-mon[103001]: pgmap v323: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:27:45.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:45 smithi017 ceph-mon[107556]: pgmap v323: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:27:45.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:45.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:45.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:45.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:45.866 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:45.928 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:45.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:45.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:46.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:46.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:46.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:46.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:46.221 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:46.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:46.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:46.446 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:27:46.446 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:27:46.446 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:27:46.446 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:27:46.446 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:27:46.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:46.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:46.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:46.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:46.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:46.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:46.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:46.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:46.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:46.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:47.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:47.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:47.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:27:47.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:47.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:47.232 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:27:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:47.244 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:47.307 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:27:47.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:27:47.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:47.346 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:27:47.347 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:27:47.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:47.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[103001]: pgmap v324: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:27:47.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2685218155' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.380 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:47] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:27:47.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[107556]: pgmap v324: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:27:47.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2685218155' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:47 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.381 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:27:47.381 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:27:47.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:47.411 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:27:47.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:27:47.443 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:27:47.443 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:27:47.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:27:47.468 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:27:47.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:47.495 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:27:47.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:27:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:47 smithi189 ceph-mon[90992]: pgmap v324: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:27:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2685218155' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:47 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:27:47.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:47.533 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:27:47.533 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:27:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:47.628 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:47.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:47.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:47.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:47.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:47.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:47.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:47.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:47.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:48.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:48.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:48.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:48.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:48.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:48.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:48.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:27:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[103001]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/869302616' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:27:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[107556]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/869302616' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:48.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:48 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:27:48.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:48 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:27:48.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:48 smithi189 ceph-mon[90992]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:48.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/869302616' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:48.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:48.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:48.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:48.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:48.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:48.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:48.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:48.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:49.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:49.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:49.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:49.212 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:49.252 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:49 smithi189 ceph-mon[90992]: pgmap v326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-09-13T14:27:49.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:49.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:49 smithi017 ceph-mon[107556]: pgmap v326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-09-13T14:27:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:49 smithi017 ceph-mon[103001]: pgmap v326: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2024-09-13T14:27:49.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:49.595 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:27:49.595 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:27:49.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:49.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:49.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:49.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:49.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:49.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:50.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:50.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:50.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:50.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:50.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:27:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:27:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:27:50.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:50.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:27:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:50.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:50.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:50.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:27:50.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:50.728 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:50.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:50.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:50.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:50.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:50.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:50.976 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:51.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:51.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:51.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:51.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:51.268 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 5 pg_num to 26 2024-09-13T14:27:51.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:51.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:51.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:51.445 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:51 smithi017 ceph-mon[103001]: pgmap v327: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:27:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:27:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:51 smithi017 ceph-mon[107556]: pgmap v327: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:27:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:27:51.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:51 smithi189 ceph-mon[90992]: pgmap v327: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:27:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:27:51.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:51.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:51.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:51.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:51.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:51.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:51.904 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:51.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:51.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:27:52.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.048 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:52.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:52.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:52.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:52.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:52.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:52.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:52.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:52.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:27:52.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:52 smithi189 ceph-mon[90992]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:52.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:27:52.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:52.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[103001]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4178662539' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[107556]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:52.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:27:52.702 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:27:52.702 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:27:52.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:52.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:52.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:52.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:52.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:52.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:53.006 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:53.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:53.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:53.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:53.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:53.218 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:53.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:53.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:53.307 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:27:53.356 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:53.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[103001]: pgmap v329: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[103001]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[107556]: pgmap v329: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:27:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:53 smithi017 ceph-mon[107556]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:53.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:53.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:53.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:53.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:53.751 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:53 smithi189 ceph-mon[90992]: pgmap v329: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:27:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:27:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:53 smithi189 ceph-mon[90992]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:53.788 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:53.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:53.855 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:27:53.855 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:27:53.905 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:27:53.905 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:27:53.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:53.954 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:27:53.955 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:27:53.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:27:53.983 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:27:53.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:53.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:54.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:27:54.019 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:27:54.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:27:54.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:27:54.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:54.098 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:27:54.098 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:27:54.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:54.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:27:54.135 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:27:54.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:54.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:54.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:54 smithi017 ceph-mon[103001]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:54.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4102819726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:54.315 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:54 smithi017 ceph-mon[107556]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:54.316 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4102819726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:54.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:54.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:54 smithi189 ceph-mon[90992]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4102819726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:27:54.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:54.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:54.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.684 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:54.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:54.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:54.812 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:54.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:54.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:54.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:55.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:55.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:55.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:55.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:55.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:55.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:55.497 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:55.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:55.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[103001]: pgmap v332: 99 pgs: 5 peering, 9 unknown, 85 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:55.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[103001]: 5.8 scrub starts 2024-09-13T14:27:55.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[103001]: 5.5 scrub starts 2024-09-13T14:27:55.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[103001]: 5.0 scrub starts 2024-09-13T14:27:55.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[103001]: 5.7 scrub starts 2024-09-13T14:27:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[107556]: pgmap v332: 99 pgs: 5 peering, 9 unknown, 85 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[107556]: 5.8 scrub starts 2024-09-13T14:27:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[107556]: 5.5 scrub starts 2024-09-13T14:27:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[107556]: 5.0 scrub starts 2024-09-13T14:27:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:55 smithi017 ceph-mon[107556]: 5.7 scrub starts 2024-09-13T14:27:55.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:55 smithi189 ceph-mon[90992]: pgmap v332: 99 pgs: 5 peering, 9 unknown, 85 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:55.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:55 smithi189 ceph-mon[90992]: 5.8 scrub starts 2024-09-13T14:27:55.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:55 smithi189 ceph-mon[90992]: 5.5 scrub starts 2024-09-13T14:27:55.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:55 smithi189 ceph-mon[90992]: 5.0 scrub starts 2024-09-13T14:27:55.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:55 smithi189 ceph-mon[90992]: 5.7 scrub starts 2024-09-13T14:27:55.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:55.654 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:27:55.654 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:27:55.654 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:27:55.654 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:27:55.654 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:27:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:55.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:55.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:55.814 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:55.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:55.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:55.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:56.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:56.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:56.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:56.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:56.229 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:56.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:56.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:56.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:56.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:56.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.6 scrub starts 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.3 scrub starts 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.5 scrub ok 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.10 scrub starts 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.0 scrub ok 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.8 scrub ok 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.17 scrub starts 2024-09-13T14:27:56.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:56 smithi189 ceph-mon[90992]: 5.18 scrub starts 2024-09-13T14:27:56.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:56.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.6 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.3 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.5 scrub ok 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.10 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.0 scrub ok 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.8 scrub ok 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.17 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[103001]: 5.18 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.6 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.3 scrub starts 2024-09-13T14:27:56.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.5 scrub ok 2024-09-13T14:27:56.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.10 scrub starts 2024-09-13T14:27:56.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.0 scrub ok 2024-09-13T14:27:56.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.8 scrub ok 2024-09-13T14:27:56.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.17 scrub starts 2024-09-13T14:27:56.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:56 smithi017 ceph-mon[107556]: 5.18 scrub starts 2024-09-13T14:27:56.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:56.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:56.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:56.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:56.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:56.960 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:27:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:56.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:56.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:57.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:57.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:57.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:57.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:57.321 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:27:57.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:57.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:27:57.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:57.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: pgmap v333: 99 pgs: 1 active+clean+scrubbing, 5 peering, 7 unknown, 86 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.7 scrub ok 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.15 scrub starts 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.6 scrub ok 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.1 scrub starts 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.10 scrub ok 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.3 scrub ok 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.16 scrub starts 2024-09-13T14:27:57.430 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: 5.18 scrub ok 2024-09-13T14:27:57.431 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/454570730' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.431 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.431 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:57.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:57.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:57.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:57.693 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:27:57.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:27:57.693 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-13T14:27:57.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:27:57] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: pgmap v333: 99 pgs: 1 active+clean+scrubbing, 5 peering, 7 unknown, 86 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.7 scrub ok 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.15 scrub starts 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.6 scrub ok 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.1 scrub starts 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.10 scrub ok 2024-09-13T14:27:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.3 scrub ok 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.16 scrub starts 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: 5.18 scrub ok 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/454570730' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: pgmap v333: 99 pgs: 1 active+clean+scrubbing, 5 peering, 7 unknown, 86 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.7 scrub ok 2024-09-13T14:27:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.15 scrub starts 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.6 scrub ok 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.1 scrub starts 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.10 scrub ok 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.3 scrub ok 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.16 scrub starts 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: 5.18 scrub ok 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/454570730' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:27:57.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:27:57.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:57.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:57.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:27:57.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:57.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:57.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:27:58.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:27:58.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:27:58.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:27:58.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:58.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:27:58.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:27:58.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:58.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:27:58.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:58.485 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: 5.1 scrub ok 2024-09-13T14:27:58.485 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: 5.17 scrub ok 2024-09-13T14:27:58.485 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: 5.15 scrub ok 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: 5.13 scrub starts 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:58.486 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:58 smithi189 ceph-mon[90992]: 5.16 scrub ok 2024-09-13T14:27:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:58.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:58.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:58.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: 5.1 scrub ok 2024-09-13T14:27:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: 5.17 scrub ok 2024-09-13T14:27:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: 5.15 scrub ok 2024-09-13T14:27:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: 5.13 scrub starts 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[103001]: 5.16 scrub ok 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: 5.1 scrub ok 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: 5.17 scrub ok 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: 5.15 scrub ok 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: 5.13 scrub starts 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:27:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:27:58.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:27:58.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:58.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:58 smithi017 ceph-mon[107556]: 5.16 scrub ok 2024-09-13T14:27:58.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:58.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:58.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:74513 5.7 5:eba6007a:::benchmark_data_smithi017_147622_object74512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:27:58.924 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 103.30318316, 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088206253999999998, 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 74513 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615766+0000", 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615827+0000", 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1118999999999995e-05 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615842+0000", 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4612e-05 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615847+0000", 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2259999999999996e-06 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615910+0000", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3670000000000005e-05 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.616014+0000", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010381 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.616146+0000", 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013156299999999999 2024-09-13T14:27:58.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.619203+0000", 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030568549999999998 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.619236+0000", 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3325999999999999e-05 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703883+0000", 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084646937000000005 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703935+0000", 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2346999999999999e-05 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703974+0000", 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8306000000000002e-05 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.930 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:74520 5.7 5:e2c73bd7:::benchmark_data_smithi017_147622_object74519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:15.623727+0000", 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 103.295223476, 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.085137947000000005, 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 74520 2024-09-13T14:27:58.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623727+0000", 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623727+0000", 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623725+0000", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623766+0000", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1343e-05 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623788+0000", 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1761000000000001e-05 2024-09-13T14:27:58.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623795+0000", 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2690000000000002e-06 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.623877+0000", 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2064999999999994e-05 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.624012+0000", 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013418600000000001 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.624163+0000", 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015102200000000001 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626593+0000", 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024302960000000002 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626605+0000", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2089000000000001e-05 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.708769+0000", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082163853999999995 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.708825+0000", 2024-09-13T14:27:58.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5727999999999999e-05 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.708865+0000", 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0379000000000001e-05 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:74522 5.7 5:e63a8001:::benchmark_data_smithi017_147622_object74521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:15.626741+0000", 2024-09-13T14:27:58.937 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 103.292209926, 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082284328000000004, 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 74522 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626741+0000", 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626741+0000", 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626739+0000", 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626799+0000", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9938000000000001e-05 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626810+0000", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1749999999999999e-05 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626815+0000", 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4780000000000002e-06 2024-09-13T14:27:58.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626854+0000", 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8940999999999999e-05 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.626940+0000", 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5959999999999997e-05 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.627033+0000", 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3089000000000003e-05 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.629131+0000", 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002098309 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.629256+0000", 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012513100000000001 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.708952+0000", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079696051000000004 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.708993+0000", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0685000000000001e-05 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.709025+0000", 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1866000000000003e-05 2024-09-13T14:27:58.943 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89287 5.7 5:edd860ce:::benchmark_data_smithi017_147622_object89286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 92.051610460999996, 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096836681999999993, 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.944 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89287 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:27:58.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867339+0000", 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867370+0000", 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0840999999999998e-05 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867379+0000", 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5699999999999999e-06 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867384+0000", 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4139999999999996e-06 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867466+0000", 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2606000000000002e-05 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867583+0000", 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011711599999999999 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867713+0000", 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013018099999999999 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871241+0000", 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035272559999999999 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876787+0000", 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055461260000000002 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964085+0000", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087298007999999996 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964136+0000", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0902999999999997e-05 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964177+0000", 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0966e-05 2024-09-13T14:27:58.950 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89292 5.7 5:e49b0fb3:::benchmark_data_smithi017_147622_object89291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 92.050133564000006, 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097664913000000006, 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.951 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89292 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868816+0000", 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868853+0000", 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6808999999999999e-05 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868860+0000", 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2400000000000001e-06 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868864+0000", 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9090000000000002e-06 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871269+0000", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024050870000000002 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871391+0000", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012149799999999999 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871488+0000", 2024-09-13T14:27:58.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6978999999999994e-05 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876857+0000", 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0053692790000000002 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879226+0000", 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002368906 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966399+0000", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087172629000000001 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966449+0000", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0000000000000002e-05 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966482+0000", 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3136999999999998e-05 2024-09-13T14:27:58.958 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89296 5.7 5:e4b5a107:::benchmark_data_smithi017_147622_object89295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 92.047575378000005, 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.095206473, 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89296 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871374+0000", 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871402+0000", 2024-09-13T14:27:58.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8283e-05 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871413+0000", 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0832000000000001e-05 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871418+0000", 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3029999999999999e-06 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874286+0000", 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028671970000000001 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874387+0000", 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101537 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874508+0000", 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121116 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879128+0000", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046201239999999998 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879232+0000", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010359199999999999 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966520+0000", 2024-09-13T14:27:58.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087287662000000002 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966556+0000", 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5957000000000001e-05 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966582+0000", 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5950000000000001e-05 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.965 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89297 5.7 5:eed05594:::benchmark_data_smithi017_147622_object89296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 92.047246884000003, 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.103594748, 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89297 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.966 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871703+0000", 2024-09-13T14:27:58.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871764+0000", 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1189e-05 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871775+0000", 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0699e-05 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871779+0000", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7319999999999998e-06 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876883+0000", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0051035389999999998 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.877015+0000", 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132323 2024-09-13T14:27:58.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.877156+0000", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000140853 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879144+0000", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019881629999999998 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879270+0000", 2024-09-13T14:27:58.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012565300000000001 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975170+0000", 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.095899897999999997 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975260+0000", 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0172000000000003e-05 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975298+0000", 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8241000000000002e-05 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89303 5.7 5:ea49a0ca:::benchmark_data_smithi017_147622_object89302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 92.039577451, 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096065179000000001, 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.972 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89303 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.973 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879372+0000", 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879408+0000", 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6168000000000003e-05 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879421+0000", 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3124e-05 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879426+0000", 2024-09-13T14:27:58.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8969999999999999e-06 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879462+0000", 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6359999999999997e-05 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879574+0000", 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111155 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879712+0000", 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013855900000000001 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.881300+0000", 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0015874369999999999 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.881428+0000", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012835799999999999 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975333+0000", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.093904986999999995 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975405+0000", 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2446000000000004e-05 2024-09-13T14:27:58.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975438+0000", 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2879000000000001e-05 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109098 5.f 5:f3c36ffb:::benchmark_data_smithi017_147622_object109097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.408583113999995, 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094563000999999994, 2024-09-13T14:27:58.979 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109098 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.980 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510366+0000", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510398+0000", 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1823999999999999e-05 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510410+0000", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2128e-05 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510415+0000", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6380000000000001e-06 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510815+0000", 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000399317 2024-09-13T14:27:58.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510936+0000", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121722 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.511065+0000", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012823000000000001 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599922+0000", 2024-09-13T14:27:58.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088857631000000006 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599955+0000", 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2291e-05 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604840+0000", 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00488551 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604891+0000", 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0939000000000001e-05 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604930+0000", 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9393e-05 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.985 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109100 5.f 5:f32deec1:::benchmark_data_smithi017_147622_object109099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:27:58.988 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:27:58.988 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.408525882999996, 2024-09-13T14:27:58.988 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094580336000000001, 2024-09-13T14:27:58.988 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109100 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.989 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510424+0000", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510504+0000", 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0269e-05 2024-09-13T14:27:58.990 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510716+0000", 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021181999999999999 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510723+0000", 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5810000000000002e-06 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.991 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515083+0000", 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043600599999999998 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515212+0000", 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012924 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515380+0000", 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016826100000000001 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.992 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599981+0000", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084601126999999998 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.603595+0000", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036140569999999999 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604941+0000", 2024-09-13T14:27:58.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0013456259999999999 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604976+0000", 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.506e-05 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605005+0000", 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8768e-05 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:58.994 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109105 5.f 5:fdd6df1a:::benchmark_data_smithi017_147622_object109104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.406066516999999, 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.092191632999999995, 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:58.995 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109105 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:27:58.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512883+0000", 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512934+0000", 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0825999999999998e-05 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:58.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512945+0000", 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1219e-05 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512950+0000", 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3600000000000004e-06 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519194+0000", 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062439180000000002 2024-09-13T14:27:58.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519309+0000", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011505 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519438+0000", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012902899999999999 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600004+0000", 2024-09-13T14:27:58.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.080566213999999997 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.603619+0000", 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003614492 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605011+0000", 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00139258 2024-09-13T14:27:59.000 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:59.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:59.002 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605046+0000", 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4820000000000002e-05 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605076+0000", 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9302000000000002e-05 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109097 5.2 e112/104 5:482ebefe:::benchmark_data_smithi017_147622_object109096:head v 112'6799, mlcod=112'6796)", 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.513310+0000", 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.405640567000006, 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.086606877999999998, 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513310+0000", 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513310+0000", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513309+0000", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513338+0000", 2024-09-13T14:27:59.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8994999999999999e-05 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513349+0000", 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0674e-05 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.513354+0000", 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1710000000000001e-06 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.516459+0000", 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031050510000000002 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.516478+0000", 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9185999999999999e-05 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599853+0000", 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083374502000000003 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.007 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599917+0000", 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4090000000000005e-05 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109104 5.2 e112/104 5:4e10141e:::benchmark_data_smithi017_147622_object109103:head v 112'6801, mlcod=112'6797)", 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.396447531000007, 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096480472999999997, 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.008 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522522+0000", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9335e-05 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522528+0000", 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9529999999999996e-06 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522531+0000", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2219999999999998e-06 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608519+0000", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.085988093000000002 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608537+0000", 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8139999999999999e-05 2024-09-13T14:27:59.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.618668+0000", 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010130251 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.618983+0000", 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00031575400000000001 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109109 5.f 5:f93ff6aa:::benchmark_data_smithi017_147622_object109108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.395663321000001, 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088054187000000006, 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109109 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523286+0000", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523318+0000", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2270000000000001e-05 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523329+0000", 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0380999999999999e-05 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523334+0000", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4380000000000003e-06 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600029+0000", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.076695248999999993 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600167+0000", 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013783299999999999 2024-09-13T14:27:59.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600306+0000", 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013934099999999999 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605082+0000", 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047755430000000001 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608456+0000", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0033741959999999999 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611258+0000", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028014479999999998 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611307+0000", 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8899000000000001e-05 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611341+0000", 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4808000000000003e-05 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109111 5.b e112/104 5:da474388:::benchmark_data_smithi017_147622_object109110:head v 112'6937, mlcod=112'6934)", 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.523418+0000", 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.395532153000005, 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083860287000000006, 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.019 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523418+0000", 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523418+0000", 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523418+0000", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523449+0000", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1197e-05 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523457+0000", 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3340000000000002e-06 2024-09-13T14:27:59.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523462+0000", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.656e-06 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523512+0000", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9663000000000002e-05 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523530+0000", 2024-09-13T14:27:59.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8698999999999998e-05 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.607198+0000", 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083667345000000004 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.607279+0000", 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0891999999999997e-05 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109114 5.e e112/104 5:7c0f1b50:::benchmark_data_smithi017_147622_object109113:head v 112'6762, mlcod=112'6760)", 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.392218525000004, 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097411384000000004, 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526731+0000", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526754+0000", 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3203000000000001e-05 2024-09-13T14:27:59.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599845+0000", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073090426 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599866+0000", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.128e-05 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599947+0000", 2024-09-13T14:27:59.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0652e-05 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599969+0000", 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.228e-05 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.623992+0000", 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024023052 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.624143+0000", 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015129400000000001 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:141157 5.7 5:e7a83112:::benchmark_data_smithi017_147622_object6528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:17.700068+0000", 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 41.218882858999997, 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082362310999999994, 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.028 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 141157 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700068+0000", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700068+0000", 2024-09-13T14:27:59.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700066+0000", 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700072+0000", 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4229999999999996e-06 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700081+0000", 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1900000000000001e-06 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700086+0000", 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8160000000000001e-06 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700142+0000", 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6372999999999997e-05 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700239+0000", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6787999999999996e-05 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.700327+0000", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7655e-05 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.771616+0000", 2024-09-13T14:27:59.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.071289012999999998 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.771641+0000", 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5299e-05 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.782264+0000", 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010622872 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.782372+0000", 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108397 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:17.782430+0000", 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7655999999999998e-05 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:158669 5.7 5:eaa5c984:::benchmark_data_smithi017_147622_object24040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:27:59.034 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 21.422458359, 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.090928470999999997, 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 158669 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:27:59.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496491+0000", 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496495+0000", 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9940000000000002e-06 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496507+0000", 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1243000000000001e-05 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496513+0000", 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511633+0000", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015120188 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511737+0000", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103869 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511851+0000", 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011382700000000001 2024-09-13T14:27:59.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.517530+0000", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005679585 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.526651+0000", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091204649999999995 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587351+0000", 2024-09-13T14:27:59.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.060699837999999999 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587386+0000", 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4789000000000002e-05 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587421+0000", 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5061999999999997e-05 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:158681 5.7 5:e5e33199:::benchmark_data_smithi017_147622_object24052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 21.408478783, 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.101405525, 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 158681 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.041 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510470+0000", 2024-09-13T14:27:59.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510477+0000", 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1130000000000003e-06 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510489+0000", 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2063e-05 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510495+0000", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5310000000000001e-06 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.517730+0000", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072353929999999997 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.521211+0000", 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003480698 2024-09-13T14:27:59.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.521315+0000", 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010391300000000001 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.526879+0000", 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005564083 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.527227+0000", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034801400000000001 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611799+0000", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084571795000000005 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611839+0000", 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0491999999999998e-05 2024-09-13T14:27:59.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611877+0000", 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8022000000000003e-05 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:158700 5.7 5:edc04566:::benchmark_data_smithi017_147622_object24071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:37.534438+0000", 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 21.384512255000001, 2024-09-13T14:27:59.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087015319999999993, 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 158700 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534438+0000", 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534438+0000", 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534437+0000", 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534443+0000", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8130000000000001e-06 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534460+0000", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7200000000000001e-05 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.534466+0000", 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2020000000000001e-06 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.538353+0000", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003887204 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.538476+0000", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012246499999999999 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.538591+0000", 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000115652 2024-09-13T14:27:59.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.546652+0000", 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080607279999999996 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.546658+0000", 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1630000000000001e-06 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:27:59.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.621337+0000", 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.074678867999999995 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.621413+0000", 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5968000000000002e-05 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.621454+0000", 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0324000000000002e-05 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.053 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.054 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.054 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:27:59.054 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:27:59.054 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:27:59.065 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:59.082 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:59.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:59.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:59.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:59.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:27:59.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:27:59.416 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:27:59.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[103001]: pgmap v335: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:59.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[103001]: 5.4 scrub starts 2024-09-13T14:27:59.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[103001]: 5.11 scrub starts 2024-09-13T14:27:59.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[103001]: 5.13 scrub ok 2024-09-13T14:27:59.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[107556]: pgmap v335: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:59.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[107556]: 5.4 scrub starts 2024-09-13T14:27:59.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[107556]: 5.11 scrub starts 2024-09-13T14:27:59.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:27:59 smithi017 ceph-mon[107556]: 5.13 scrub ok 2024-09-13T14:27:59.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:59.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:59.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:27:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:59.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:27:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:59 smithi189 ceph-mon[90992]: pgmap v335: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail 2024-09-13T14:27:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:59 smithi189 ceph-mon[90992]: 5.4 scrub starts 2024-09-13T14:27:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:59 smithi189 ceph-mon[90992]: 5.11 scrub starts 2024-09-13T14:27:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:27:59 smithi189 ceph-mon[90992]: 5.13 scrub ok 2024-09-13T14:27:59.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:27:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:27:59.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:27:59.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:27:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:27:59.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:27:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:00.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:00.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:00.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:00.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:00.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:00.351 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:00.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:00.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:00.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[103001]: 5.4 scrub ok 2024-09-13T14:28:00.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[103001]: 5.11 scrub ok 2024-09-13T14:28:00.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[103001]: 5.9 scrub starts 2024-09-13T14:28:00.490 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:28:00.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[107556]: 5.4 scrub ok 2024-09-13T14:28:00.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[107556]: 5.11 scrub ok 2024-09-13T14:28:00.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[107556]: 5.9 scrub starts 2024-09-13T14:28:00.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:28:00.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:00.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:00.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:00.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:00.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:00.744 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:00 smithi189 ceph-mon[90992]: 5.4 scrub ok 2024-09-13T14:28:00.744 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:00 smithi189 ceph-mon[90992]: 5.11 scrub ok 2024-09-13T14:28:00.744 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:00 smithi189 ceph-mon[90992]: 5.9 scrub starts 2024-09-13T14:28:00.744 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:28:00.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:00.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:00.920 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:00.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:00.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:28:01.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:28:01.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:01.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:01.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.204 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:01.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:01.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:01.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:01.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:01.351 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: 5.9 scrub ok 2024-09-13T14:28:01.396 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:01.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:01.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:01.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:01.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:01.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: 5.9 scrub ok 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: pgmap v336: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: 5.2 scrub starts 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: 5.14 scrub starts 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[103001]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: 5.9 scrub ok 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: pgmap v336: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: 5.2 scrub starts 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: 5.14 scrub starts 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:28:01.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:01 smithi017 ceph-mon[107556]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:28:01.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:01.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: pgmap v336: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: 5.2 scrub starts 2024-09-13T14:28:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: 5.14 scrub starts 2024-09-13T14:28:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:28:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:01 smithi189 ceph-mon[90992]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:28:01.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:01.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:01.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:01.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:01.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:02.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:02.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:02.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:02.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:02.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:02.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:02.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:02.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:02.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:02.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:02.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: 5.2 scrub ok 2024-09-13T14:28:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: 5.12 scrub starts 2024-09-13T14:28:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: 5.14 scrub ok 2024-09-13T14:28:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: 5.19 deep-scrub starts 2024-09-13T14:28:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: 5.12 scrub ok 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: 5.2 scrub ok 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: 5.12 scrub starts 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: 5.14 scrub ok 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: 5.19 deep-scrub starts 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: 5.12 scrub ok 2024-09-13T14:28:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:28:02.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: 5.2 scrub ok 2024-09-13T14:28:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: 5.12 scrub starts 2024-09-13T14:28:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: 5.14 scrub ok 2024-09-13T14:28:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: 5.19 deep-scrub starts 2024-09-13T14:28:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:28:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: 5.12 scrub ok 2024-09-13T14:28:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:28:02.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:02.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:02.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:02.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:02.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:03.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:03.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:03.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:03.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:28:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:28:03.181 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:28:03.181 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:28:03.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:03.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:03.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:03.371 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:03 smithi189 ceph-mon[90992]: pgmap v339: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:03.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:03.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:03.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:03.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:03.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[107556]: pgmap v339: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:03.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:28:03.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[107556]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[107556]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:28:03.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[107556]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[103001]: pgmap v339: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:28:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:28:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[103001]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[103001]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:28:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:03 smithi017 ceph-mon[103001]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:03.701 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:03.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:28:03.715 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:28:03.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:28:03.737 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:28:03.755 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:28:03.755 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:28:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:28:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:03 smithi189 ceph-mon[90992]: 5.10 starting backfill to osd.6 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:03 smithi189 ceph-mon[90992]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:28:03.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:03 smithi189 ceph-mon[90992]: 5.10 starting backfill to osd.7 from (0'0,0'0] MAX to 121'11020 2024-09-13T14:28:03.773 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:28:03.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:28:03.794 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:28:03.794 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:28:03.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:03.816 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:28:03.816 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:28:03.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:03.834 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:28:03.835 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:28:03.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:28:03.854 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:28:03.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:03.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:03.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:04.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:04.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:04.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:04.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:04.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:04.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:04.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:04.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:04.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:04.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.618 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:04.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:04.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:04 smithi189 ceph-mon[90992]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:28:04.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1837695570' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:04.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:04 smithi189 ceph-mon[90992]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:28:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[103001]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:28:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1837695570' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[103001]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:28:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[107556]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:28:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1837695570' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:04 smithi017 ceph-mon[107556]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:28:04.717 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:28:04.717 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:28:04.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:04.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:04.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:04.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:04.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:04.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:05.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.043 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:05.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:05.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:05.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:05.278 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:05.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:05.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:28:05.419 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:05.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:05.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:05.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:05 smithi017 ceph-mon[103001]: pgmap v342: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:05 smithi017 ceph-mon[103001]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 123'10926 2024-09-13T14:28:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:05 smithi017 ceph-mon[107556]: pgmap v342: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:05 smithi017 ceph-mon[107556]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 123'10926 2024-09-13T14:28:05.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:05 smithi189 ceph-mon[90992]: pgmap v342: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:05 smithi189 ceph-mon[90992]: 5.11 starting backfill to osd.6 from (0'0,0'0] MAX to 123'10926 2024-09-13T14:28:05.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:05.852 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:05.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:05.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:05.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:05.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:06.130 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:06.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:06.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:06.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:28:06.302 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:28:06.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-13T14:28:06.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:06.416 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:06.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:06.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:06.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:06.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:06.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:06.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:06.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:06.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:07.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:07.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:07.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:07.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:07.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:07.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:07.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:07.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:07.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:07] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-09-13T14:28:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:07 smithi017 ceph-mon[103001]: pgmap v344: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:07 smithi017 ceph-mon[107556]: pgmap v344: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:07.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:07.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:07.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:07.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:07 smithi189 ceph-mon[90992]: pgmap v344: 99 pgs: 1 active+clean+scrubbing+deep, 2 remapped+peering, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2024-09-13T14:28:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:07.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:07.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:07.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:07.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:07.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:07.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:08.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:08.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:08.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.220 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:08.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:08.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:08.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:08.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:08.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:08.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:08.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.646 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:08.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:08.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:08.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:08.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:08.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:08.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:09.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:09.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:09.141 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:28:09.141 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:28:09.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:09.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:09.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:09.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:09.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:09.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:09.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:09.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:28:09.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.620 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:09.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:09.656 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:09 smithi017 ceph-mon[107556]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3212/265800 objects misplaced (1.208%); 12 MiB/s, 191 objects/s recovering 2024-09-13T14:28:09.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:09 smithi017 ceph-mon[103001]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3212/265800 objects misplaced (1.208%); 12 MiB/s, 191 objects/s recovering 2024-09-13T14:28:09.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:09 smithi189 ceph-mon[90992]: pgmap v345: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3212/265800 objects misplaced (1.208%); 12 MiB/s, 191 objects/s recovering 2024-09-13T14:28:09.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:09.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:09.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:09.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:28:09.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:09.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:09.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:10.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:10.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:10.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:10.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:10.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:10.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:10.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:10.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:10.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:28:10.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:10.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:10.682 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:28:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:28:10.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:10.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:10.821 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.854 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:10.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:10.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:10.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:10.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:11.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:11.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:11.152 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:11.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:11.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:11.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:11.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:11.427 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:28:11.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:11.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: pgmap v346: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5856/261630 objects misplaced (2.238%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: pgmap v346: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5856/261630 objects misplaced (2.238%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:28:11.582 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:11.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:11.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:11.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: pgmap v346: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5856/261630 objects misplaced (2.238%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:28:11.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:28:11.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:28:11.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:11.771 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:28:11.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:11.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:11.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:11.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:11.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:11.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:12.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:12.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:12.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:12.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:12.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:12.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:12.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:12.441 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:12.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:12.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:12.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:12.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:12.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:12.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]} : dispatch 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]} : dispatch 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]} : dispatch 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:28:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]} : dispatch 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]} : dispatch 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]} : dispatch 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:28:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4157271922' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]} : dispatch 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]} : dispatch 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]} : dispatch 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:28:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:12.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:12.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:12.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:12.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:12.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:13.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:13.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:13.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:13.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:13.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:13.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:13.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:13.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: pgmap v349: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5856/257895 objects misplaced (2.271%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]}]': finished 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]}]': finished 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]}]': finished 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:28:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: 5.19 deep-scrub ok 2024-09-13T14:28:13.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:13 smithi189 ceph-mon[90992]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:28:13.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:13.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:13.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:13.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:13.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: pgmap v349: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5856/257895 objects misplaced (2.271%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:28:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]}]': finished 2024-09-13T14:28:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: 5.19 deep-scrub ok 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[103001]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: pgmap v349: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 96 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5856/257895 objects misplaced (2.271%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.b", "id": [5, 1]}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.f", "id": [5, 3]}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.12", "id": [5, 6]}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:28:13.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: 5.19 deep-scrub ok 2024-09-13T14:28:13.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:13 smithi017 ceph-mon[107556]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:28:13.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:13.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:13.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:13.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:13.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:14.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:14.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:14.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:14.194 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:28:14.194 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:28:14.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:14.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:14.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:14.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:14.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:14.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:14.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:14.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:14.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:14.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:14.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:14.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:14.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:14.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:28:14.880 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:28:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:28:14.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:28:14.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:15.016 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:28:15.016 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:28:15.016 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:28:15.017 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:28:15.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:15.048 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:28:15.048 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:28:15.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:28:15.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:28:15.113 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:28:15.113 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:28:15.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:15.155 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:28:15.155 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:28:15.185 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:28:15.185 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:28:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.218 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:15.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:15.221 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:28:15.221 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:28:15.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.364 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:15.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: pgmap v352: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 95 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2756/252864 objects misplaced (1.090%); 28 MiB/s, 447 objects/s recovering 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.f starting backfill to osd.3 from (0'0,0'0] MAX to 127'11468 2024-09-13T14:28:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.b starting backfill to osd.1 from (0'0,0'0] MAX to 127'11759 2024-09-13T14:28:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1103246421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: pgmap v352: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 95 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2756/252864 objects misplaced (1.090%); 28 MiB/s, 447 objects/s recovering 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.f starting backfill to osd.3 from (0'0,0'0] MAX to 127'11468 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.b starting backfill to osd.1 from (0'0,0'0] MAX to 127'11759 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1103246421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:15.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:15.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:15.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:15.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:15.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:15.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: pgmap v352: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 95 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2756/252864 objects misplaced (1.090%); 28 MiB/s, 447 objects/s recovering 2024-09-13T14:28:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.12 starting backfill to osd.1 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.13 starting backfill to osd.1 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.f starting backfill to osd.3 from (0'0,0'0] MAX to 127'11468 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.b starting backfill to osd.1 from (0'0,0'0] MAX to 127'11759 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.12 starting backfill to osd.3 from (0'0,0'0] MAX to 126'11150 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: 5.13 starting backfill to osd.3 from (0'0,0'0] MAX to 127'11194 2024-09-13T14:28:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1103246421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.789 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:15.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:15.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:15.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:15.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:15.989 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:16.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:16.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:16.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:16.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:16.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:16.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:28:16.317 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:28:16.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-13T14:28:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:16.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:16.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:16.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:16.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:16.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:16.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:16.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:16.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:16.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:16.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:16.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:17.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:17.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:17.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:17.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:17.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:17.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:17.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:17.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:17.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:17.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:17 smithi017 ceph-mon[103001]: pgmap v354: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 94 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 8096/251481 objects misplaced (3.219%); 26 MiB/s, 410 objects/s recovering 2024-09-13T14:28:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:17] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T14:28:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:17 smithi017 ceph-mon[107556]: pgmap v354: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 94 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 8096/251481 objects misplaced (3.219%); 26 MiB/s, 410 objects/s recovering 2024-09-13T14:28:17.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:17 smithi189 ceph-mon[90992]: pgmap v354: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 1 remapped+peering, 94 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 8096/251481 objects misplaced (3.219%); 26 MiB/s, 410 objects/s recovering 2024-09-13T14:28:17.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:17.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:17.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:17.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:18.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:18.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:18.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:18.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:18.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:18.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:18.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:18.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:18.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:18.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:18.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:18.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:18.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:18.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:18.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:18.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:19.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:19.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:19.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:19.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:19.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:19.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:19.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:189975 5.16 e131/119 5:6da6d777:::benchmark_data_smithi017_147622_object55346:head v 131'11225, mlcod=131'11223)", 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:19.333587+0000", 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.010183131, 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010210303, 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:28:19.346 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333587+0000", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333587+0000", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333587+0000", 2024-09-13T14:28:19.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333592+0000", 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4299999999999997e-06 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333604+0000", 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1525999999999999e-05 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333609+0000", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9200000000000003e-06 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.333656+0000", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7275999999999999e-05 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336617+0000", 2024-09-13T14:28:19.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029614250000000002 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:189979 5.16 e131/119 5:6c15ad48:::benchmark_data_smithi017_147622_object55350:head v 131'11226, mlcod=131'11223)", 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:19.336837+0000", 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0069332739999999997, 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070293439999999999, 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336837+0000", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336837+0000", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336837+0000", 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:19.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336844+0000", 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1550000000000001e-06 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336854+0000", 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.077e-05 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.336860+0000", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5790000000000003e-06 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.340856+0000", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039957789999999997 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.340875+0000", 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9621999999999999e-05 2024-09-13T14:28:19.353 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:189982 5.1 e131/119 5:82b3da3e:::benchmark_data_smithi017_147622_object55353:head v 131'11312, mlcod=131'11310)", 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:19.341312+0000", 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0024588790000000002, 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002609349, 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341312+0000", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341312+0000", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341311+0000", 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341317+0000", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9769999999999997e-06 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341325+0000", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7789999999999996e-06 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341330+0000", 2024-09-13T14:28:19.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7350000000000004e-06 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341369+0000", 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9342999999999999e-05 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341386+0000", 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6991e-05 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:19.357 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:189983 5.16 e131/119 5:6e3037df:::benchmark_data_smithi017_147622_object55354:head v 131'11227, mlcod=131'11223)", 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:19.341398+0000", 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.002373015, 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002587532, 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341398+0000", 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341398+0000", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341397+0000", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341402+0000", 2024-09-13T14:28:19.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9840000000000001e-06 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341419+0000", 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7058000000000002e-05 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341424+0000", 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.13e-06 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.343238+0000", 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0018138080000000001 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.343254+0000", 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6087999999999999e-05 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.361 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:189984 5.9 5:95ef208f:::benchmark_data_smithi017_147622_object55355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:19.341545+0000", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0022252439999999999, 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025057930000000001, 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 189984 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:19.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341545+0000", 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341545+0000", 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341544+0000", 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341550+0000", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6590000000000004e-06 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341563+0000", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3073e-05 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.341569+0000", 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9440000000000002e-06 2024-09-13T14:28:19.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.342403+0000", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00083428099999999998 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.342516+0000", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000112864 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:19.342940+0000", 2024-09-13T14:28:19.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042451700000000001 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 5 2024-09-13T14:28:19.366 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:28:19.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:19.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:19.508 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:19 smithi189 ceph-mon[90992]: pgmap v355: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 93 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 18226/249408 objects misplaced (7.308%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:28:19.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:19.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:19.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:19 smithi017 ceph-mon[103001]: pgmap v355: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 93 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 18226/249408 objects misplaced (7.308%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:28:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:19 smithi017 ceph-mon[107556]: pgmap v355: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+clean+scrubbing+deep, 93 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 18226/249408 objects misplaced (7.308%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:28:19.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:19.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:19.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:19.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:19.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:19.944 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:19.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:19.983 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:20.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:20.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:20.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.188 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:20.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:20.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:20.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:28:20.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:20.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:20.470 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:20.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:20.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:20.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:20.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:20.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:20.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:20.908 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:20.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:20.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:21.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:21.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:21.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:21.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:21.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:21.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:21.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:21.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:21.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:21.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:21.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:21.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[103001]: pgmap v356: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20703/244665 objects misplaced (8.462%); 23 MiB/s, 363 objects/s recovering 2024-09-13T14:28:21.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[107556]: pgmap v356: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20703/244665 objects misplaced (8.462%); 23 MiB/s, 363 objects/s recovering 2024-09-13T14:28:21.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:21 smithi189 ceph-mon[90992]: pgmap v356: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 20703/244665 objects misplaced (8.462%); 23 MiB/s, 363 objects/s recovering 2024-09-13T14:28:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:21.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:21.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:21.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:28:21.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:21.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:22.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:22.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:22.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:22.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:22.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:22.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:22.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:22.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:22.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:22.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:22.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:22.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:22.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:22.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:22.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:22.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:22.950 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:28:22.950 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:28:22.951 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:28:22.951 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:28:22.951 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:28:22.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:23.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:23.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:23.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:23.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:23.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:23.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:23.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:23.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:23.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:23.586 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:23.670 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:23 smithi189 ceph-mon[90992]: pgmap v357: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 20367/239955 objects misplaced (8.488%); 20 MiB/s, 317 objects/s recovering 2024-09-13T14:28:23.670 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:23 smithi189 ceph-mon[90992]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:23.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:23 smithi017 ceph-mon[103001]: pgmap v357: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 20367/239955 objects misplaced (8.488%); 20 MiB/s, 317 objects/s recovering 2024-09-13T14:28:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:23 smithi017 ceph-mon[103001]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:23 smithi017 ceph-mon[107556]: pgmap v357: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 20367/239955 objects misplaced (8.488%); 20 MiB/s, 317 objects/s recovering 2024-09-13T14:28:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:23 smithi017 ceph-mon[107556]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:23.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:28:23.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:23.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:23.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:23.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:23.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:24.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:24.077 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:28:24.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:24.111 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:24.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:24.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:24.264 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:24.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:24.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:24.406 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:28:24.472 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:28:24.473 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:28:24.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:24.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:24 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1410841383' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:24 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:24.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:24.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:24 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1410841383' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:24 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:24 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1410841383' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:24 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:24.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:24.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:24.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:24.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:24.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:25.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:25.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:25.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:25.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:25.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:25.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:25.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:25.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:25.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:25 smithi189 ceph-mon[90992]: pgmap v359: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 18903/230196 objects misplaced (8.212%); 17 MiB/s, 279 objects/s recovering 2024-09-13T14:28:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:25 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:25.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:25.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:25 smithi189 ceph-mon[90992]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:25.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.618 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:25.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[103001]: pgmap v359: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 18903/230196 objects misplaced (8.212%); 17 MiB/s, 279 objects/s recovering 2024-09-13T14:28:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[103001]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[107556]: pgmap v359: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 18903/230196 objects misplaced (8.212%); 17 MiB/s, 279 objects/s recovering 2024-09-13T14:28:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:25 smithi017 ceph-mon[107556]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:25.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:25.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:25.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:28:25.792 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:28:25.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-13T14:28:25.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:25.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:25.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:25.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:26.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:26.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:26.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:26.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:26.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:26.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:26.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:26.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:26.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:26.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:26.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:26.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:26.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:26.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:26.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:26.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:27.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:27.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:27.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:27.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:27.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:27.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:27.412 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:28:27.412 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:28:27.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:28:27.412 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:31124 5.4 5:25c34b92:::benchmark_data_smithi017_147622_object31123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:25:51.223994+0000", 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 156.18438204200001, 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.092360098000000002, 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.413 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 31124 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.223994+0000", 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.223994+0000", 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.223993+0000", 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.224121+0000", 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012805100000000001 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.224130+0000", 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2010000000000005e-06 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.224135+0000", 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6149999999999999e-06 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.224772+0000", 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00063693199999999999 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.224892+0000", 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012045900000000001 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:28:27.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.225024+0000", 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132006 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226248+0000", 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001223189 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.227587+0000", 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001339146 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.316284+0000", 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088696811 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.316331+0000", 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7678e-05 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.316354+0000", 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3006999999999998e-05 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.421 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:31129 5.4 5:2c58ab6c:::benchmark_data_smithi017_147622_object31128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:25:51.226402+0000", 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 156.18197443899999, 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.092557324999999996, 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 31129 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.422 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226402+0000", 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226402+0000", 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226400+0000", 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226449+0000", 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8831000000000001e-05 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226463+0000", 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3919e-05 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226469+0000", 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2210000000000002e-06 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226515+0000", 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5389000000000002e-05 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226629+0000", 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000114481 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.226775+0000", 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014611299999999999 2024-09-13T14:28:27.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.227726+0000", 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00095094299999999999 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.228083+0000", 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035701199999999998 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.318897+0000", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.090813789000000006 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.318927+0000", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9768999999999999e-05 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:25:51.318959+0000", 2024-09-13T14:28:27.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2088999999999997e-05 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178191 5.10 5.9646f730 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.411290+0000", 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.997085771999998, 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95798324999999995, 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.429 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178191 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411290+0000", 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411290+0000", 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411289+0000", 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411294+0000", 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.442e-06 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411302+0000", 2024-09-13T14:28:27.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7540000000000006e-06 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411306+0000", 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.143e-06 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411478+0000", 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017183399999999999 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.411485+0000", 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0090000000000001e-06 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369265+0000", 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.957780454 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369274+0000", 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6039999999999993e-06 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178264 5.10 5.70e43830 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.478734+0000", 2024-09-13T14:28:27.435 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.929642241, 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89056739600000001, 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178264 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478734+0000", 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478734+0000", 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478732+0000", 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478739+0000", 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3289999999999999e-06 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478750+0000", 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1863e-05 2024-09-13T14:28:27.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478756+0000", 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8499999999999999e-06 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478800+0000", 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3201000000000002e-05 2024-09-13T14:28:27.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.478807+0000", 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4540000000000001e-06 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369295+0000", 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89048818100000005 2024-09-13T14:28:27.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369301+0000", 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1480000000000003e-06 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178269 5.10 5.a40d8310 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.441 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.483195+0000", 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.925181131999999, 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88613200000000003, 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178269 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483195+0000", 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483195+0000", 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483193+0000", 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483201+0000", 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.216e-06 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483213+0000", 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2139e-05 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483219+0000", 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.835e-06 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483264+0000", 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5806999999999997e-05 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.483271+0000", 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9539999999999998e-06 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369321+0000", 2024-09-13T14:28:27.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88604961800000004 2024-09-13T14:28:27.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369327+0000", 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0310000000000004e-06 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178320 5.10 5.4ebb7510 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.448 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.532841+0000", 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.875535475, 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.836511699, 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178320 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.449 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532841+0000", 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532841+0000", 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532839+0000", 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532845+0000", 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4539999999999998e-06 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532855+0000", 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8609999999999993e-06 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532860+0000", 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1449999999999999e-06 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532904+0000", 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4274999999999999e-05 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.532912+0000", 2024-09-13T14:28:27.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8280000000000003e-06 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369346+0000", 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83643424300000002 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369352+0000", 2024-09-13T14:28:27.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1040000000000003e-06 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178458 5.10 5.1a577c30 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.659926+0000", 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.748450111, 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70945273799999997, 2024-09-13T14:28:27.456 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178458 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659926+0000", 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659926+0000", 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659925+0000", 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659931+0000", 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2750000000000001e-06 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659943+0000", 2024-09-13T14:28:27.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2143999999999999e-05 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.659949+0000", 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.387e-06 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.661310+0000", 2024-09-13T14:28:27.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0013600730000000001 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.661316+0000", 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5050000000000004e-06 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369372+0000", 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70805567999999997 2024-09-13T14:28:27.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369379+0000", 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.058e-06 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.462 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178464 5.10 5.d3b16df0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.667176+0000", 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.741200297999999, 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70222846999999999, 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178464 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667176+0000", 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667176+0000", 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667174+0000", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667184+0000", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4599999999999992e-06 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667196+0000", 2024-09-13T14:28:27.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2367e-05 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667202+0000", 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1789999999999996e-06 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667328+0000", 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012583900000000001 2024-09-13T14:28:27.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.667335+0000", 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4119999999999998e-06 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369398+0000", 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70206354800000004 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369404+0000", 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0499999999999997e-06 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178489 5.10 5.406d12f0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.688067+0000", 2024-09-13T14:28:27.468 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.720309446999998, 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.68136336799999997, 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178489 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.469 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688067+0000", 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688067+0000", 2024-09-13T14:28:27.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688065+0000", 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688071+0000", 2024-09-13T14:28:27.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2389999999999999e-06 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688082+0000", 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0316000000000001e-05 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688087+0000", 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4450000000000004e-06 2024-09-13T14:28:27.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688126+0000", 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8955000000000002e-05 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.688133+0000", 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0530000000000001e-06 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369424+0000", 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.68129086500000002 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369430+0000", 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0569999999999998e-06 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178511 5.10 5.856ac530 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.711392+0000", 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.696984135000001, 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.65806282900000002, 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.475 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178511 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711392+0000", 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711392+0000", 2024-09-13T14:28:27.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711391+0000", 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711395+0000", 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1840000000000001e-06 2024-09-13T14:28:27.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711405+0000", 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8160000000000005e-06 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711411+0000", 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5509999999999999e-06 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711455+0000", 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4471000000000002e-05 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.711463+0000", 2024-09-13T14:28:27.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2490000000000004e-06 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369449+0000", 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.65798635500000002 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369455+0000", 2024-09-13T14:28:27.480 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9209999999999997e-06 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178635 5.19 5:9d29e85a:::benchmark_data_smithi017_147622_object44006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.857808+0000", 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.550568510000002, 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.099626707999999994, 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.481 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178635 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857808+0000", 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857808+0000", 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857806+0000", 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857813+0000", 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2020000000000001e-06 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857833+0000", 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0513000000000002e-05 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.857840+0000", 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6599999999999998e-06 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.871118+0000", 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013278092999999999 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.871269+0000", 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015078699999999999 2024-09-13T14:28:27.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.871612+0000", 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034344900000000001 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.873162+0000", 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0015497499999999999 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957124+0000", 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083962116000000003 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957358+0000", 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00023373600000000001 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957398+0000", 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0769000000000002e-05 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957434+0000", 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5948000000000002e-05 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.488 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178639 5.9 5:91de3484:::benchmark_data_smithi017_147622_object44010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.859042+0000", 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.549333914999998, 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.098248473000000003, 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.489 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178639 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859042+0000", 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859042+0000", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859041+0000", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859046+0000", 2024-09-13T14:28:27.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0159999999999999e-06 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859061+0000", 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4234e-05 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.859066+0000", 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7660000000000004e-06 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.873075+0000", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014008994 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.873161+0000", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5782e-05 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.873539+0000", 2024-09-13T14:28:27.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00037761400000000002 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.876071+0000", 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00253206 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957135+0000", 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.081063707999999998 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957194+0000", 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9837000000000003e-05 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957247+0000", 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2657000000000001e-05 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.957291+0000", 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3594000000000003e-05 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178700 5.10 5.c4373710 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.987669+0000", 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.420707533000002, 2024-09-13T14:28:27.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.38181140800000002, 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178700 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987669+0000", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987669+0000", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987667+0000", 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:28:27.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987676+0000", 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4399999999999994e-06 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987697+0000", 2024-09-13T14:28:27.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0613000000000001e-05 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987704+0000", 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.126e-06 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987782+0000", 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7977000000000002e-05 2024-09-13T14:28:27.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.987790+0000", 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4170000000000003e-06 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369474+0000", 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.38168417300000002 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369480+0000", 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3409999999999999e-06 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178706 5.10 5.7660f750 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.992520+0000", 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.415856555000001, 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.37698897799999997, 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178706 2024-09-13T14:28:27.503 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992520+0000", 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992520+0000", 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992518+0000", 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992525+0000", 2024-09-13T14:28:27.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.46e-06 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992538+0000", 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3560000000000001e-05 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992544+0000", 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.908e-06 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992592+0000", 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7367999999999998e-05 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.992600+0000", 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7930000000000003e-06 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369502+0000", 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.376902073 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369509+0000", 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2009999999999996e-06 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178709 5.10 5.9d595e10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:00.997898+0000", 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.410478433000002, 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.371636625, 2024-09-13T14:28:27.509 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178709 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.510 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997898+0000", 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997898+0000", 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997896+0000", 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997903+0000", 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7100000000000001e-06 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997917+0000", 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3871e-05 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.997922+0000", 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.699e-06 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.998124+0000", 2024-09-13T14:28:27.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000202222 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:00.998134+0000", 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.006e-05 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369528+0000", 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.37139382399999998 2024-09-13T14:28:27.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369534+0000", 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0530000000000003e-06 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178728 5.19 5:9b44fe1a:::benchmark_data_smithi017_147622_object44099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:01.038142+0000", 2024-09-13T14:28:27.515 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.370233952, 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12165256200000001, 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178728 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.516 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038142+0000", 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038142+0000", 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.517 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038140+0000", 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038149+0000", 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1480000000000006e-06 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.518 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038168+0000", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9854e-05 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.038176+0000", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1029999999999996e-06 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.074564+0000", 2024-09-13T14:28:27.519 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.036388763999999997 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.075059+0000", 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00049423000000000002 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.075705+0000", 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00064689399999999996 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.520 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.081993+0000", 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062872730000000003 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.082021+0000", 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.794e-05 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.159687+0000", 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.077665921999999998 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.159747+0000", 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0560000000000003e-05 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.159795+0000", 2024-09-13T14:28:27.522 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7644e-05 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:178755 5.10 5.d3829d70 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-09-13T14:28:27.523 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:01.160003+0000", 2024-09-13T14:28:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:27.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.248373090000001, 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.209557982, 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.525 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 178755 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160003+0000", 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.526 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160003+0000", 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160002+0000", 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160007+0000", 2024-09-13T14:28:27.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8949999999999997e-06 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160016+0000", 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3899999999999999e-06 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160020+0000", 2024-09-13T14:28:27.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8260000000000003e-06 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160055+0000", 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4477000000000002e-05 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.160062+0000", 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3359999999999997e-06 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369555+0000", 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.209492492 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:01.369561+0000", 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5259999999999999e-06 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179094 5.19 5:980140b0:::benchmark_data_smithi017_147622_object44465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.597217+0000", 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 24.811158990999999, 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11769724199999999, 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179094 2024-09-13T14:28:27.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597217+0000", 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597217+0000", 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597215+0000", 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597222+0000", 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5379999999999999e-06 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597237+0000", 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5143000000000001e-05 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.597243+0000", 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.534 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.619004+0000", 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021761346000000001 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.619094+0000", 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9493000000000005e-05 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.619436+0000", 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000342512 2024-09-13T14:28:27.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.622577+0000", 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003140641 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.646843+0000", 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024265770999999998 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.714836+0000", 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.067993676000000003 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.714877+0000", 2024-09-13T14:28:27.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0447000000000001e-05 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.714914+0000", 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7787999999999998e-05 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.538 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179105 5.9 5:942e4ff2:::benchmark_data_smithi017_147622_object44476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.613876+0000", 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 24.794499930000001, 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.101152168, 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.539 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179105 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613876+0000", 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613876+0000", 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613875+0000", 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613880+0000", 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9509999999999997e-06 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613891+0000", 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1287999999999999e-05 2024-09-13T14:28:27.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.613898+0000", 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3269999999999997e-06 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.618722+0000", 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048246360000000002 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.618849+0000", 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000126539 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.619260+0000", 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041107999999999997 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.622481+0000", 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003220957 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.647126+0000", 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024644822 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.714969+0000", 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.067843710000000002 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.715000+0000", 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0865999999999999e-05 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.715028+0000", 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8291000000000001e-05 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179394 5.19 5:9e1de4e1:::benchmark_data_smithi017_147622_object44765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.939975+0000", 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 24.468401655000001, 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11635693799999999, 2024-09-13T14:28:27.547 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179394 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939975+0000", 2024-09-13T14:28:27.548 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939975+0000", 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939973+0000", 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:28:27.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939980+0000", 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.686e-06 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939991+0000", 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.168e-05 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.939997+0000", 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0469999999999999e-06 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.971713+0000", 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031715437999999999 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.971837+0000", 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000124387 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.972196+0000", 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035914399999999999 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976375+0000", 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041790259999999997 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.994454+0000", 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018078556999999999 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:28:27.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.056249+0000", 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.061795124 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.056292+0000", 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3291999999999998e-05 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.056331+0000", 2024-09-13T14:28:27.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9183000000000001e-05 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:28:27.555 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:28:27.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:27.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:27] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T14:28:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:27 smithi017 ceph-mon[107556]: pgmap v361: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18620/225645 objects misplaced (8.252%); 21 MiB/s, 329 objects/s recovering 2024-09-13T14:28:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:27 smithi017 ceph-mon[103001]: pgmap v361: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18620/225645 objects misplaced (8.252%); 21 MiB/s, 329 objects/s recovering 2024-09-13T14:28:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:27 smithi189 ceph-mon[90992]: pgmap v361: 99 pgs: 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 18620/225645 objects misplaced (8.252%); 21 MiB/s, 329 objects/s recovering 2024-09-13T14:28:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:27.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:27.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:27.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:27.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:27.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:27.973 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:28.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:28.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:28.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:28.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:28.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:28:28.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:28.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:28.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:28.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:28.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:28:28.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:28.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:28.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:28.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:28.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:28.967 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:28.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:28.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:29.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:29.101 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:28:29.102 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:28:29.123 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:28:29.123 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:28:29.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:29.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:29.156 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:28:29.157 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:28:29.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:28:29.191 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:28:29.227 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:28:29.227 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:28:29.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:29.261 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:28:29.261 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:28:29.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:29.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:28:29.299 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:28:29.329 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:28:29.329 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:28:29.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:29.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:29.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:29.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:29.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:29.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:29 smithi017 ceph-mon[103001]: pgmap v362: 99 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 17330/221211 objects misplaced (7.834%); 9.4 MiB/s, 151 objects/s recovering 2024-09-13T14:28:29.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3966739053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:29.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:29 smithi017 ceph-mon[107556]: pgmap v362: 99 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 17330/221211 objects misplaced (7.834%); 9.4 MiB/s, 151 objects/s recovering 2024-09-13T14:28:29.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3966739053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:29.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:29 smithi189 ceph-mon[90992]: pgmap v362: 99 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 17330/221211 objects misplaced (7.834%); 9.4 MiB/s, 151 objects/s recovering 2024-09-13T14:28:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3966739053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:29.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:29.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:29.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:29.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:29.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:29.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:30.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:30.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:30.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:30.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:30.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:30.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:30.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:30.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:30.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:30.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:30.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:30.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:30.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:30.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:30.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:30.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:30.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:30.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:31.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:31.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:31.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:31.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:31.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:31.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:31.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:31.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:31.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:31.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:31 smithi017 ceph-mon[103001]: pgmap v363: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 16419/215316 objects misplaced (7.626%); 15 MiB/s, 239 objects/s recovering 2024-09-13T14:28:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:31 smithi017 ceph-mon[107556]: pgmap v363: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 16419/215316 objects misplaced (7.626%); 15 MiB/s, 239 objects/s recovering 2024-09-13T14:28:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:31.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:31 smithi189 ceph-mon[90992]: pgmap v363: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 16419/215316 objects misplaced (7.626%); 15 MiB/s, 239 objects/s recovering 2024-09-13T14:28:31.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:31.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:31.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:31.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:31.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:31.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:31.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:32.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:32.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:32.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:32.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:32.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:32.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:32.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:32.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:32.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:32.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:32.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:32.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:32.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:32.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:32.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:32.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:33.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:33.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:33.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:33.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:33.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:33.400 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:33.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:33.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:33.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:33 smithi017 ceph-mon[103001]: pgmap v364: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16061/210465 objects misplaced (7.631%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:33 smithi017 ceph-mon[107556]: pgmap v364: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16061/210465 objects misplaced (7.631%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:33 smithi189 ceph-mon[90992]: pgmap v364: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 16061/210465 objects misplaced (7.631%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:33.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:33.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:33.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:33.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:33.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:34.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:34.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:34.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:34.155 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:34.159 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:34.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:34.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:34.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:34.363 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:34.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:34.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:34.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:34.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:34.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:28:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:28:34.665 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:28:34.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:34.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:34.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:34.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:34.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:34.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:35.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:35.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:35.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:35.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:35.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:35.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:35.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:35.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:35.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:35.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:35.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: pgmap v365: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 14602/200244 objects misplaced (7.292%); 13 MiB/s, 201 objects/s recovering 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: pgmap v365: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 14602/200244 objects misplaced (7.292%); 13 MiB/s, 201 objects/s recovering 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:35.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: pgmap v365: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 14602/200244 objects misplaced (7.292%); 13 MiB/s, 201 objects/s recovering 2024-09-13T14:28:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:35.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:35.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:35.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:35.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:28:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:35.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:35.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:35.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:36.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:36.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:36.140 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:36.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:36.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:36.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:36.416 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:36.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:36.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:36.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:36.688 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:36.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:36.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:36.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:36.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:36.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:28:36.953 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:28:36.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-13T14:28:36.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:37.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:37.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:37.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:37.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:37.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:37.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:37.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:37 smithi017 ceph-mon[103001]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 14278/195513 objects misplaced (7.303%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:28:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:37] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T14:28:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:37 smithi017 ceph-mon[107556]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 14278/195513 objects misplaced (7.303%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:28:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:37 smithi189 ceph-mon[90992]: pgmap v366: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 14278/195513 objects misplaced (7.303%); 11 MiB/s, 176 objects/s recovering 2024-09-13T14:28:37.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:37.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:37.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:37.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:37.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:37.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:38.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:38.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:38.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:38.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:38.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:38.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:38.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:38.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:38.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:38.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:38.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:38.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:38.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:38.788 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:38.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:38.833 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:28:38.833 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:28:38.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:38.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:38.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:39.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:39.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:39.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:39.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:39.271 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:39.274 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:39.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:39.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:39.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:39.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:39.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:39.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:39 smithi017 ceph-mon[103001]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 13642/192141 objects misplaced (7.100%); 10 MiB/s, 167 objects/s recovering 2024-09-13T14:28:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:39 smithi017 ceph-mon[107556]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 13642/192141 objects misplaced (7.100%); 10 MiB/s, 167 objects/s recovering 2024-09-13T14:28:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:39 smithi189 ceph-mon[90992]: pgmap v367: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 13642/192141 objects misplaced (7.100%); 10 MiB/s, 167 objects/s recovering 2024-09-13T14:28:39.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:39.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:39.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:39.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:39.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:40.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:40.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:40.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:40.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:40.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:40.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:40.443 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:28:40.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:40.472 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:28:40.472 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:28:40.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:28:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:40.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:40.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:40.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:40.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:40.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:40.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:40.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:41.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:41.060 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:41.162 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:41.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:41.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:41.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:41.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:41.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:41.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[103001]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12872/184839 objects misplaced (6.964%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:28:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:28:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[103001]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[107556]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12872/184839 objects misplaced (6.964%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:28:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:28:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:41 smithi017 ceph-mon[107556]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:41.711 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:41 smithi189 ceph-mon[90992]: pgmap v368: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12872/184839 objects misplaced (6.964%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:28:41.711 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:41 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:28:41.711 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/841937534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:28:41.712 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:41 smithi189 ceph-mon[90992]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:41.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:41.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:41.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:41.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:41.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:42.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:42.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:42.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:42.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:42.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:42.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:42.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:42.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:42.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:42.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:42.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:28:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:42.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:42.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:42.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:42.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:42.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:42.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:42.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:42.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:42.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:42.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:42.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:42.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:43.082 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:43.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:43.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:43.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:43.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:43.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:43.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:43.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:43.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:43.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:43.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:43.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:43 smithi017 ceph-mon[107556]: pgmap v370: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12537/179970 objects misplaced (6.966%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:28:43.712 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:43 smithi017 ceph-mon[103001]: pgmap v370: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12537/179970 objects misplaced (6.966%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:28:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:43 smithi189 ceph-mon[90992]: pgmap v370: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12537/179970 objects misplaced (6.966%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:28:43.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:43.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:43.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:43.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:43.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:43.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:44.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:44.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:44.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:44.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:44.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:44.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:44.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:44.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:44.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:44.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:44.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:44 smithi189 ceph-mon[90992]: pgmap v371: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1886278678' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:44.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:44.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:44 smithi017 ceph-mon[103001]: pgmap v371: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:44.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1886278678' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:44.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:44 smithi017 ceph-mon[107556]: pgmap v371: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:44.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1886278678' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:44.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:44.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:44.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:45.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:45.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:45.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:45.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:45.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:45.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:45.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:28:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:28:45.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:28:45.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:28:45.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:28:45.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:45.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:45.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:45.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:28:45.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:45.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:45.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:45.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:45.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:45.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:45.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:46.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:46.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:46.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:46.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:46.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:46.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:46.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:46.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:46.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:46.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:46.683 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:46.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:46.766 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:46.767 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:28:46.767 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:28:46.768 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:28:46.768 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:28:46.768 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:28:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:28:46.770 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:28:46.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:46.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:46.918 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:46.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:46 smithi017 ceph-mon[103001]: pgmap v372: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:46 smithi017 ceph-mon[107556]: pgmap v372: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:47.020 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:47.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:47.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:47.167 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:47.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:46 smithi189 ceph-mon[90992]: pgmap v372: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 11214/169593 objects misplaced (6.612%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:28:47.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:47.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:47.388 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:47] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-13T14:28:47.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:47.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:47.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:47.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:47.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:47.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:47.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:47.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:47.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:47.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:47.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:47.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:47.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:48.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:48.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:48.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:47 smithi017 ceph-mon[103001]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:48.122 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:47 smithi017 ceph-mon[107556]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:48.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:28:48.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:48.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:48.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:47 smithi189 ceph-mon[90992]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:48.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:48.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:48.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:48.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:28:48.583 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:28:48.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-13T14:28:48.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:48.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:48.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:48.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:48.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:48.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:48.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:48.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:49.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:49.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:49.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:49.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:48 smithi017 ceph-mon[103001]: pgmap v374: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 10058/161493 objects misplaced (6.228%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:28:49.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:48 smithi017 ceph-mon[103001]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:49.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:48 smithi017 ceph-mon[107556]: pgmap v374: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 10058/161493 objects misplaced (6.228%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:28:49.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:48 smithi017 ceph-mon[107556]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:49.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:49.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:49.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:48 smithi189 ceph-mon[90992]: pgmap v374: 99 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 10058/161493 objects misplaced (6.228%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:28:49.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:48 smithi189 ceph-mon[90992]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:49.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:49.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:49.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:49.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:49.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:49.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:49.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:49.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:49.950 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:49.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:49.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:49.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:50.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:50.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:50.235 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:50.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:50.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.407 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:50.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:28:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:28:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:28:50.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:50.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:50.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:50.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:50.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:50.797 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:50.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:50.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:50.936 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:50.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:51.037 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:51.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:50 smithi017 ceph-mon[103001]: pgmap v376: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 9530/155253 objects misplaced (6.138%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:51.068 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:50 smithi017 ceph-mon[107556]: pgmap v376: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 9530/155253 objects misplaced (6.138%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:51.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:51.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:51.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:51.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:50 smithi189 ceph-mon[90992]: pgmap v376: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 9530/155253 objects misplaced (6.138%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:28:51.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:51.364 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:51.466 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:51.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:51.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:51.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:51.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:51.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:51.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:51.918 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:51.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:51.919 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:28:51.919 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:28:51.919 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:28:51.919 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:28:51.919 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:28:52.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:52.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:52.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:52.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:52.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:52.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:52.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:52.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:52.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:52.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:52.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:52.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:52.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:52.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:52.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:52.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:52.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.028 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:53.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:53.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:28:53.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:53.154 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:52 smithi017 ceph-mon[103001]: pgmap v377: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 9261/151266 objects misplaced (6.122%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:28:53.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:52 smithi017 ceph-mon[107556]: pgmap v377: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 9261/151266 objects misplaced (6.122%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:28:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:53.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:53.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:52 smithi189 ceph-mon[90992]: pgmap v377: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 9261/151266 objects misplaced (6.122%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:28:53.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:53.342 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:53.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:53.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:53.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:53.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:53.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:53.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:53.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:53.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:53.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:53.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:54.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:54.043 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:54.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:54.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:54.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:54.315 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:54.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:54.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:54.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:54.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:54.688 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:54.715 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:54.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:54.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:54.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:54.891 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:28:54.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:55.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:55.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[103001]: pgmap v378: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7265/141696 objects misplaced (5.127%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:28:55.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/559130224' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[107556]: pgmap v378: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7265/141696 objects misplaced (5.127%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:28:55.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/559130224' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:54 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:55.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:55.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:55.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:54 smithi189 ceph-mon[90992]: pgmap v378: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7265/141696 objects misplaced (5.127%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:28:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/559130224' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:54 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:28:55.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:55.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:55.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:55.398 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:55.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:55.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:55.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:55.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:55.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:55.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:55.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:55.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:55.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[103001]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:28:56.061 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:28:56.062 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:56.062 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:55 smithi017 ceph-mon[107556]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:56.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:56.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:56.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:55 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:28:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:55 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:28:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:55 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:28:56.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:55 smithi189 ceph-mon[90992]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:56.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:56.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:56.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:56.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:56.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:56.641 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:56.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:28:56.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:56.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:56.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:56.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:56.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:56.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:56.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:57.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:57.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:57.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:56 smithi017 ceph-mon[103001]: pgmap v380: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7265/141696 objects misplaced (5.127%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:28:57.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:56 smithi017 ceph-mon[107556]: pgmap v380: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7265/141696 objects misplaced (5.127%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:28:57.195 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:28:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.224 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:57.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:28:57.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:56 smithi189 ceph-mon[90992]: pgmap v380: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7265/141696 objects misplaced (5.127%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:28:57.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:57.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:57.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:28:57.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:28:57.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:57.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:57.659 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:28:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:28:57] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:28:57.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:57.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:57.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:57.926 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:57.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:58.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:28:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:58.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:28:58.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:28:58.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:28:58.128 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:28:58.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:58.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:58.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:28:58.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1303156290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:58.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:28:58.172 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:28:58.173 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:58.173 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:58.173 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:28:58.173 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1303156290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:58.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:28:58.209 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:28:58.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:28:58.219 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:28:58.261 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:28:58.261 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:28:58.294 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:28:58.294 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:28:58.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:58.329 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:28:58.329 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:28:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:58.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:58.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:28:58.361 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:28:58.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:28:58.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:28:58.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:58.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:58.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:28:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:28:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1303156290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:28:58.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:58.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:58.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:58.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:58.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:58.854 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:58.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:58.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:58.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:59.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:59.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[103001]: pgmap v381: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 5644/134400 objects misplaced (4.199%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:28:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:28:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[103001]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[103001]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[107556]: pgmap v381: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 5644/134400 objects misplaced (4.199%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:28:59.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:28:59.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[107556]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:59.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:28:59 smithi017 ceph-mon[107556]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:59.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:28:59.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:28:59.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:59.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:59.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:59.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:59 smithi189 ceph-mon[90992]: pgmap v381: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 5644/134400 objects misplaced (4.199%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:28:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:28:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:59 smithi189 ceph-mon[90992]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:28:59 smithi189 ceph-mon[90992]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:59.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:28:59.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:28:59.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:28:59.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:28:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:28:59.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:28:59.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:28:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:28:59.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:28:59.933 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:28:59.933 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:29:00.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:00.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:00.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:00.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:00.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:00.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:00.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:00.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:00.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:00.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:00 smithi189 ceph-mon[90992]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 137'12420 2024-09-13T14:29:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:00 smithi189 ceph-mon[90992]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:29:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:29:00.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:00.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:00.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:00.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[103001]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 137'12420 2024-09-13T14:29:00.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[103001]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:29:00.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:29:00.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[107556]: 5.14 starting backfill to osd.3 from (0'0,0'0] MAX to 137'12420 2024-09-13T14:29:00.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[107556]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:29:00.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:29:00.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:00.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:00.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:00.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:00.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:00.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:00.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:01.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:01.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:01.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:01.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:01.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:01.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:01.394 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:01.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:01.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:01.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:01.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:01.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[103001]: pgmap v385: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 5644/127968 objects misplaced (4.410%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:29:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:29:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[103001]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:29:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[107556]: pgmap v385: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 5644/127968 objects misplaced (4.410%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:29:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:29:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:01 smithi017 ceph-mon[107556]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:29:01.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:01 smithi189 ceph-mon[90992]: pgmap v385: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 5644/127968 objects misplaced (4.410%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:29:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:29:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:01 smithi189 ceph-mon[90992]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:29:01.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:01.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:01.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:01.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:01.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:02.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:02.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:02.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:02.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:02.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:02.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:02.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:02.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:02.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:29:02.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:02.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:02.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:02.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:02 smithi017 ceph-mon[103001]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:29:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:29:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:02 smithi017 ceph-mon[107556]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:29:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:29:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:02 smithi189 ceph-mon[90992]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:29:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:29:02.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:02.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:02.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:02.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:02.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:03.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:03.195 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:03.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:03.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:03.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:03.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:03.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:03.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:03.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[103001]: pgmap v388: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 5880/124191 objects misplaced (4.735%); 14 MiB/s, 218 objects/s recovering 2024-09-13T14:29:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:29:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[103001]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:29:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[103001]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 140'12552 2024-09-13T14:29:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[103001]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:29:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[107556]: pgmap v388: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 5880/124191 objects misplaced (4.735%); 14 MiB/s, 218 objects/s recovering 2024-09-13T14:29:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:29:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[107556]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:29:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[107556]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 140'12552 2024-09-13T14:29:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:03 smithi017 ceph-mon[107556]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:29:03.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:03.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:29:03.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:03 smithi189 ceph-mon[90992]: pgmap v388: 99 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 95 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 5880/124191 objects misplaced (4.735%); 14 MiB/s, 218 objects/s recovering 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:03 smithi189 ceph-mon[90992]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:03 smithi189 ceph-mon[90992]: 5.15 starting backfill to osd.5 from (0'0,0'0] MAX to 140'12552 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:03 smithi189 ceph-mon[90992]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:29:03.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:03.845 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:03.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:03.968 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:03.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:04.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:04.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:04.189 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:04.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:04.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:04.228 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:04.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:04.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:04.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:04.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:04.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:04.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:04.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:04.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:04.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:04.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:04.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:04.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:05.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:05.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:05.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:05.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:05.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:05.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:05.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:05.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:05.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:05.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:05.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:05.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[103001]: pgmap v391: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3040/119196 objects misplaced (2.550%); 33 MiB/s, 529 objects/s recovering 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[103001]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[103001]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[103001]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[107556]: pgmap v391: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3040/119196 objects misplaced (2.550%); 33 MiB/s, 529 objects/s recovering 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[107556]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:29:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[107556]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:05 smithi017 ceph-mon[107556]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:05 smithi189 ceph-mon[90992]: pgmap v391: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3040/119196 objects misplaced (2.550%); 33 MiB/s, 529 objects/s recovering 2024-09-13T14:29:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:05 smithi189 ceph-mon[90992]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:29:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:05 smithi189 ceph-mon[90992]: 5.16 starting backfill to osd.1 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:05 smithi189 ceph-mon[90992]: 5.16 starting backfill to osd.3 from (0'0,0'0] MAX to 141'12412 2024-09-13T14:29:05.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:05.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:05.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:05.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:05.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:06.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:06.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:06.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:06.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:06.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:06.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:06.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:06.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:06.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:06.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:06.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:06.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:06.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:06.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:06.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:06.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:06.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:07.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:07.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:07.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:07.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:07.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:07.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:07.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:29:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:29:07.451 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:29:07.451 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:29:07.451 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:29:07.451 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:29:07.451 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:29:07.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:07.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:07.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:07 smithi017 ceph-mon[103001]: pgmap v393: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 3040/119196 objects misplaced (2.550%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:29:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:07] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:29:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:07 smithi017 ceph-mon[107556]: pgmap v393: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 3040/119196 objects misplaced (2.550%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:29:07.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:07.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:07 smithi189 ceph-mon[90992]: pgmap v393: 99 pgs: 2 peering, 1 remapped+peering, 2 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 3040/119196 objects misplaced (2.550%); 19 MiB/s, 298 objects/s recovering 2024-09-13T14:29:07.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:07.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:07.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:07.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:08.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:08.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:08.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:08.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:08.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:08.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:08.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:08.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:08.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:08.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:08.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:08.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:08.821 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:08.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:08.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:08.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:08.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:09.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:09.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:09.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:09.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:09.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:09.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:09.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:09.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:09.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:09.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:09 smithi017 ceph-mon[103001]: pgmap v394: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 2 peering, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 2030/115560 objects misplaced (1.757%); 35 MiB/s, 567 objects/s recovering 2024-09-13T14:29:09.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:09 smithi017 ceph-mon[103001]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:29:09.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:09 smithi017 ceph-mon[107556]: pgmap v394: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 2 peering, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 2030/115560 objects misplaced (1.757%); 35 MiB/s, 567 objects/s recovering 2024-09-13T14:29:09.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:09 smithi017 ceph-mon[107556]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:29:09.646 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:09.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:09 smithi189 ceph-mon[90992]: pgmap v394: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 2 peering, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 2030/115560 objects misplaced (1.757%); 35 MiB/s, 567 objects/s recovering 2024-09-13T14:29:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:09 smithi189 ceph-mon[90992]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:29:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:09.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:09.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:09.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:09.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:10.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:10.057 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:10.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:10.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:10.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:29:10.195 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:29:10.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-13T14:29:10.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:10.416 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:29:10.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:10.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:10 smithi189 ceph-mon[90992]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:29:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:29:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/240469354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:10.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[103001]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:29:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:29:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/240469354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[107556]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:29:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:29:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/240469354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:29:10.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:10.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:10.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:10.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:10.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:10.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:10.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:11.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:11.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:11.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:11.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:11.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:11.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:11.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:11.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:11.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:11.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[103001]: pgmap v397: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 4366/110235 objects misplaced (3.961%); 22 MiB/s, 348 objects/s recovering 2024-09-13T14:29:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:29:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[103001]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[107556]: pgmap v397: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 4366/110235 objects misplaced (3.961%); 22 MiB/s, 348 objects/s recovering 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:29:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:11 smithi017 ceph-mon[107556]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:29:11.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:11 smithi189 ceph-mon[90992]: pgmap v397: 99 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 4366/110235 objects misplaced (3.961%); 22 MiB/s, 348 objects/s recovering 2024-09-13T14:29:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:11 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:29:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:29:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:11 smithi189 ceph-mon[90992]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:29:11.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:11.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:11.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:11.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:11.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:11.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:12.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:12.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:12.138 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:29:12.139 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:29:12.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:12.166 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:29:12.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:29:12.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:29:12.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:29:12.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:29:12.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:29:12.230 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:29:12.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:29:12.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:12.250 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:29:12.250 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:29:12.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:29:12.272 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:29:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:12.290 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:29:12.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:29:12.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:12.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:12.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:12.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[103001]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:29:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3707820730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[107556]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:29:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3707820730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:12 smithi189 ceph-mon[90992]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:29:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3707820730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:12.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:12.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:12.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:12.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:12.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:13.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:13.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:13.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:13.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:13.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:13.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:13.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:13.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:13.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:13.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:13.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:13.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[103001]: pgmap v400: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3487/106566 objects misplaced (3.272%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:29:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[103001]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[103001]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[103001]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:29:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[107556]: pgmap v400: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3487/106566 objects misplaced (3.272%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:29:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[107556]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[107556]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:13 smithi017 ceph-mon[107556]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:29:13.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:13 smithi189 ceph-mon[90992]: pgmap v400: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3487/106566 objects misplaced (3.272%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:29:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:13 smithi189 ceph-mon[90992]: 5.17 starting backfill to osd.3 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:13 smithi189 ceph-mon[90992]: 5.17 starting backfill to osd.7 from (0'0,0'0] MAX to 147'12750 2024-09-13T14:29:13.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:13 smithi189 ceph-mon[90992]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:29:13.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:13.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:13.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:13.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:13.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:14.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:14.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:14.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:14.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:14.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:14.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:14.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:14.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:14.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:14.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:14.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:14.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:14.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:14.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:14.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:15.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:15.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:15.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:15.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:15.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:15.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:15.404 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:15.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:15.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:15.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:15.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:15.608 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:15 smithi017 ceph-mon[103001]: pgmap v402: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 2135/100443 objects misplaced (2.126%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:29:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:15 smithi017 ceph-mon[107556]: pgmap v402: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 2135/100443 objects misplaced (2.126%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:29:15.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:15.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:29:15.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:15 smithi189 ceph-mon[90992]: pgmap v402: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 2135/100443 objects misplaced (2.126%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:29:15.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:15.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:15.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:15.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:16.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:16.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:16.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:16.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:16.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:239179 5.c 5:3d250b06:::benchmark_data_smithi017_147622_object104550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.348066+0000", 2024-09-13T14:29:16.371 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.020313154999999999, 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020344088, 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 239179 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348066+0000", 2024-09-13T14:29:16.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348066+0000", 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348068+0000", 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.86e-06 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348075+0000", 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2300000000000002e-06 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348091+0000", 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5452999999999999e-05 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348099+0000", 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6780000000000006e-06 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348189+0000", 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9402999999999998e-05 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348325+0000", 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013676399999999999 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.348693+0000", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036741200000000001 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.359593+0000", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010899842999999999 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.359670+0000", 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7682000000000006e-05 2024-09-13T14:29:16.377 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:239185 5.c 5:360d3f75:::benchmark_data_smithi017_147622_object104556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.352020+0000", 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.016359167000000001, 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016468773999999999, 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:29:16.378 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 239185 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352020+0000", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352020+0000", 2024-09-13T14:29:16.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352022+0000", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4899999999999999e-06 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352027+0000", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1850000000000003e-06 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352040+0000", 2024-09-13T14:29:16.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3444e-05 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.352048+0000", 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2570000000000007e-06 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.355537+0000", 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003488896 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.355657+0000", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120079 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.355772+0000", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114831 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.360241+0000", 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044690609999999999 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365242+0000", 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050003599999999997 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:239193 5.5 e150/119 5:a47b2876:::benchmark_data_smithi017_147622_object104564:head v 150'12867, mlcod=150'12865)", 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.361596+0000", 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.006782761, 2024-09-13T14:29:16.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006963075, 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361596+0000", 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361596+0000", 2024-09-13T14:29:16.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361598+0000", 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.178e-06 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361606+0000", 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8510000000000005e-06 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361626+0000", 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9451000000000001e-05 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361634+0000", 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4179999999999997e-06 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361736+0000", 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010193599999999999 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.361756+0000", 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0137999999999999e-05 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:239192 5.7 e150/119 5:e548e3d0:::benchmark_data_smithi017_147622_object104563:head v 150'12877, mlcod=150'12875)", 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.362202+0000", 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0061768860000000004, 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064066009999999996, 2024-09-13T14:29:16.387 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362202+0000", 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362202+0000", 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362203+0000", 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8000000000000005e-07 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362210+0000", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.294e-06 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362222+0000", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1308000000000001e-05 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362229+0000", 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0520000000000004e-06 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362301+0000", 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2648e-05 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.362320+0000", 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8986000000000001e-05 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:239195 5.b e150/148 5:ddd6b650:::benchmark_data_smithi017_147622_object104566:head v 150'15245, mlcod=150'15243)", 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.364002+0000", 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0043774, 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046551830000000002, 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364002+0000", 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364002+0000", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364003+0000", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9500000000000001e-07 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364010+0000", 2024-09-13T14:29:16.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5290000000000003e-06 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364023+0000", 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2843000000000001e-05 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364030+0000", 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1389999999999998e-06 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364110+0000", 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9462999999999995e-05 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.364130+0000", 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0488000000000001e-05 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35034.0:239196 5.7 e150/119 5:e5588a4e:::benchmark_data_smithi017_147622_object104567:head v 150'12878, mlcod=150'12875)", 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:16.365894+0000", 2024-09-13T14:29:16.395 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0024849210000000002, 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028100220000000001, 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365894+0000", 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365894+0000", 2024-09-13T14:29:16.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365895+0000", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7999999999999995e-07 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365901+0000", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0630000000000002e-06 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365911+0000", 2024-09-13T14:29:16.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0389e-05 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.365929+0000", 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7737e-05 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.366566+0000", 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00063673699999999996 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:29:16.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:16.366586+0000", 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9966000000000002e-05 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 6 2024-09-13T14:29:16.399 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:29:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:16.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:16.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:16.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:16.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:16.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:16.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:16.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:16.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:16.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:29:16.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:16.931 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:16.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:17.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:17.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:17.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:17.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:17.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:17.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:17.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:17.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:17.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:17.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:17.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:29:17.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:17.646 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:17] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:29:17.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:17 smithi017 ceph-mon[103001]: pgmap v403: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 2135/100443 objects misplaced (2.126%); 14 MiB/s, 222 objects/s recovering 2024-09-13T14:29:17.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:17 smithi017 ceph-mon[103001]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:29:17.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:17 smithi017 ceph-mon[107556]: pgmap v403: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 2135/100443 objects misplaced (2.126%); 14 MiB/s, 222 objects/s recovering 2024-09-13T14:29:17.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:17 smithi017 ceph-mon[107556]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:29:17.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:17.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:17 smithi189 ceph-mon[90992]: pgmap v403: 99 pgs: 1 remapped+peering, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 2135/100443 objects misplaced (2.126%); 14 MiB/s, 222 objects/s recovering 2024-09-13T14:29:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:17 smithi189 ceph-mon[90992]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:29:17.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:17.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:17.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:17.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:17.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:18.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:18.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:18.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:18.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:18.200 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:18.223 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:18.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:18.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:18.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:18.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:18.471 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:18.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:18.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:18 smithi017 ceph-mon[103001]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:29:18.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:29:18.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:18 smithi017 ceph-mon[107556]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:29:18.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:29:18.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:18.670 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:18.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:18 smithi189 ceph-mon[90992]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:29:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:29:18.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:18.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:18.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:18.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:18.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:18.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:19.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:19.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:19.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:19.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:19.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:29:19.395 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:29:19.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:19.466 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:19.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:19.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:19.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[103001]: pgmap v406: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3096/94806 objects misplaced (3.266%); 17 MiB/s, 270 objects/s recovering 2024-09-13T14:29:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:29:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[103001]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:29:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[107556]: pgmap v406: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3096/94806 objects misplaced (3.266%); 17 MiB/s, 270 objects/s recovering 2024-09-13T14:29:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:29:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:19 smithi017 ceph-mon[107556]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:29:19.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:19 smithi189 ceph-mon[90992]: pgmap v406: 99 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 96 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3096/94806 objects misplaced (3.266%); 17 MiB/s, 270 objects/s recovering 2024-09-13T14:29:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:29:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:19 smithi189 ceph-mon[90992]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:29:19.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:19.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:19.946 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:19.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:19.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:20.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:20.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:20.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:20.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:20.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:20.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:20.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:20.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:20.480 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:20.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:20.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:20.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:20.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:20 smithi189 ceph-mon[90992]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:29:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:20 smithi189 ceph-mon[90992]: pgmap v409: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 2094/88665 objects misplaced (2.362%); 23 MiB/s, 365 objects/s recovering 2024-09-13T14:29:20.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:29:20.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:20.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[103001]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:29:20.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[103001]: pgmap v409: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 2094/88665 objects misplaced (2.362%); 23 MiB/s, 365 objects/s recovering 2024-09-13T14:29:20.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:29:20.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[107556]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:29:20.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[107556]: pgmap v409: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 981 op/s; 2094/88665 objects misplaced (2.362%); 23 MiB/s, 365 objects/s recovering 2024-09-13T14:29:20.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:29:20.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:20.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:20.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:20.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:21.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:21.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:21.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:21.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:21.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:21.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:21.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:21.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:21.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:21.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:29:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[103001]: 5.18 starting backfill to osd.4 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[103001]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:29:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[103001]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:29:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[107556]: 5.18 starting backfill to osd.4 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[107556]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:29:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:21 smithi017 ceph-mon[107556]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:29:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:21 smithi189 ceph-mon[90992]: 5.18 starting backfill to osd.4 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:21 smithi189 ceph-mon[90992]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:29:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:21 smithi189 ceph-mon[90992]: 5.18 starting backfill to osd.6 from (0'0,0'0] MAX to 152'12846 2024-09-13T14:29:21.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:29:21.831 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:29:21.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-13T14:29:21.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:21.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:21.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:21.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:22.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:22.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:22.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:22.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:22.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:22.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:22.522 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:22.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:22.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:22.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:22 smithi189 ceph-mon[90992]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:29:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:22 smithi189 ceph-mon[90992]: pgmap v412: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2094/86865 objects misplaced (2.411%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:29:22.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:22.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:22.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:22.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:22 smithi017 ceph-mon[103001]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:29:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:22 smithi017 ceph-mon[103001]: pgmap v412: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2094/86865 objects misplaced (2.411%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:29:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:22 smithi017 ceph-mon[107556]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:29:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:22 smithi017 ceph-mon[107556]: pgmap v412: 99 pgs: 2 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2094/86865 objects misplaced (2.411%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:29:23.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:23.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:23.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:23.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:23.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:23.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:23.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:23.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:23.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:23 smithi189 ceph-mon[90992]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:29:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:23 smithi189 ceph-mon[90992]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:23 smithi189 ceph-mon[90992]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:23.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:23.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[103001]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:29:23.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[103001]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[103001]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[107556]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:29:23.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[107556]: 5.19 starting backfill to osd.1 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:23 smithi017 ceph-mon[107556]: 5.19 starting backfill to osd.5 from (0'0,0'0] MAX to 153'15710 2024-09-13T14:29:23.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:23.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:23.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:24.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:24.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:24.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:24.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:24.233 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:24.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:24.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:24.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:24.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:24.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:24.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:24 smithi017 ceph-mon[103001]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:29:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:24 smithi017 ceph-mon[103001]: pgmap v415: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 1706/83715 objects misplaced (2.038%); 30 MiB/s, 472 objects/s recovering 2024-09-13T14:29:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:24 smithi017 ceph-mon[107556]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:29:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:24 smithi017 ceph-mon[107556]: pgmap v415: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 1706/83715 objects misplaced (2.038%); 30 MiB/s, 472 objects/s recovering 2024-09-13T14:29:24.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:24.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:24.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:24.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:24.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:24.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:24 smithi189 ceph-mon[90992]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:29:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:24 smithi189 ceph-mon[90992]: pgmap v415: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 1706/83715 objects misplaced (2.038%); 30 MiB/s, 472 objects/s recovering 2024-09-13T14:29:25.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:25.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:25.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:25.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:25.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:25.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:25.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:25.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:25.454 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:25.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:25.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:25.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:25.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:25.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:25.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:25.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:25.978 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:26.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:26.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:26.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:26.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:26.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:26.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:26.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:26.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:26.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:26.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:26.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:26.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:26.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:26.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:26.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:26.875 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:29:26.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:27.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[103001]: pgmap v416: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 1706/83715 objects misplaced (2.038%); 22 MiB/s, 354 objects/s recovering 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2105556007' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[107556]: pgmap v416: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 1706/83715 objects misplaced (2.038%); 22 MiB/s, 354 objects/s recovering 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2105556007' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.094 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:29:27.094 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:29:27.095 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:29:27.095 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:29:27.095 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:29:27.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:27.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:26 smithi189 ceph-mon[90992]: pgmap v416: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 309 op/s; 1706/83715 objects misplaced (2.038%); 22 MiB/s, 354 objects/s recovering 2024-09-13T14:29:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2105556007' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:27.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:27.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:27.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:27.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:27.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:27.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:27.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:27.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:27.702 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:27] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:29:27.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:27.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:27.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:27.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:27.982 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:28.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:28.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:28.151 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[103001]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[107556]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:29:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:28.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:28.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:27 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:27 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:27 smithi189 ceph-mon[90992]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:29:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:28.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:28.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:28.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:28.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:28.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:28.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:28.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:28.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:28.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:28.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:28.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:28.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:28.973 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:28.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:29.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:29.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:28 smithi017 ceph-mon[103001]: pgmap v418: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 1706/79746 objects misplaced (2.139%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:29:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:28 smithi017 ceph-mon[107556]: pgmap v418: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 1706/79746 objects misplaced (2.139%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:29:29.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:29.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:28 smithi189 ceph-mon[90992]: pgmap v418: 99 pgs: 1 active+remapped+backfilling, 1 peering, 97 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 1706/79746 objects misplaced (2.139%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:29:29.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:29.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:29.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:29.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:29.587 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:29.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:29.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:29.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:29.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:29.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:29.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:29.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:30.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:30.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:30.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:30.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:29 smithi017 ceph-mon[103001]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:29:30.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:29 smithi017 ceph-mon[107556]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:29:30.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:29 smithi189 ceph-mon[90992]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:29:30.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:30.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:30.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:30.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:30.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:30.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:30.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:30.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:30.701 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:30.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:30.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:30.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:30.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:30 smithi017 ceph-mon[103001]: pgmap v420: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 222/71802 objects misplaced (0.309%); 16 MiB/s, 258 objects/s recovering 2024-09-13T14:29:30.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:30 smithi017 ceph-mon[103001]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:29:30.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:30 smithi017 ceph-mon[107556]: pgmap v420: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 222/71802 objects misplaced (0.309%); 16 MiB/s, 258 objects/s recovering 2024-09-13T14:29:30.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:30 smithi017 ceph-mon[107556]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:29:30.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:30.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:31.054 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:29:31.054 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:29:31.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:31.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:29:31.097 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:29:31.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:31.136 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:29:31.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:29:31.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:29:31.170 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:29:31.200 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:29:31.200 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:29:31.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:31.234 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:29:31.234 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:29:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:30 smithi189 ceph-mon[90992]: pgmap v420: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 222/71802 objects misplaced (0.309%); 16 MiB/s, 258 objects/s recovering 2024-09-13T14:29:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:30 smithi189 ceph-mon[90992]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:29:31.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:31.269 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:29:31.269 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:29:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:31.303 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:29:31.304 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:29:31.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:31.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:31.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:31.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:31.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:31.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:29:31.706 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:29:31.707 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:29:31.707 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-13T14:29:31.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:31.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:31.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:31.954 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:31.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:32.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:29:32.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:32.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2326589482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2326589482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:32.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2326589482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:32.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:32.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:32.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:32.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:32.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:32.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:32.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:32.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:32.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:29:32.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:32.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:32.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:32.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:32.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:33.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:33.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:33.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:32 smithi017 ceph-mon[103001]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 222/68529 objects misplaced (0.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:29:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:32 smithi017 ceph-mon[107556]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 222/68529 objects misplaced (0.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:29:33.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:33.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:32 smithi189 ceph-mon[90992]: pgmap v422: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 222/68529 objects misplaced (0.324%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:29:33.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:33.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:33.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:33.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:33.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:33.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:29:33.725 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:29:33.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:33.814 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:33.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:33.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:33.877 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:33.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:33.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:33.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:34.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:34.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:34.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:34.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:34.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:34.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.516 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:34.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:34.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:34.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:34.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:34.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:34.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:34.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:35.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:35.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:29:35.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:35.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:34 smithi017 ceph-mon[103001]: pgmap v423: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7.1 MiB/s, 114 objects/s recovering 2024-09-13T14:29:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:34 smithi017 ceph-mon[107556]: pgmap v423: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7.1 MiB/s, 114 objects/s recovering 2024-09-13T14:29:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.214 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:35.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:35.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:34 smithi189 ceph-mon[90992]: pgmap v423: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7.1 MiB/s, 114 objects/s recovering 2024-09-13T14:29:35.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:35.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:35.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:35.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:35.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:35.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:35.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:35.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:35.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:35.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:35.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:35.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:35.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:36.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:36.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:36.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:36.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:36.258 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:36.258 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.258 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:36.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:36.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:36.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:36.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:36.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:36.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:36.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:36.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:36.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:36.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:36.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:36.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:37.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:37.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:37.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:37.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:37.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:37.185 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:36 smithi017 ceph-mon[103001]: pgmap v424: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:29:37.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:36 smithi017 ceph-mon[107556]: pgmap v424: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:29:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:36 smithi189 ceph-mon[90992]: pgmap v424: 99 pgs: 99 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:29:37.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:37.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:37.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:37] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T14:29:37.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:37.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.596 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:37.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:37.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:37.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:37.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:37.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:37.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:37.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:38.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:38.123 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.154 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:38.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:38.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:38.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:38.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:38.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:38.470 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:29:38.470 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:29:38.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:38.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:38.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:38.725 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:29:38.726 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:29:38.726 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:29:38.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:29:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:38.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:38.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:38.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:38.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:38.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:38.969 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:29:38.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:39.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:39.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:39.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:38 smithi189 ceph-mon[90992]: pgmap v425: 99 pgs: 99 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 770 KiB/s, 12 objects/s recovering 2024-09-13T14:29:39.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/599918873' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:38 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[103001]: pgmap v425: 99 pgs: 99 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 770 KiB/s, 12 objects/s recovering 2024-09-13T14:29:39.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/599918873' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.270 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[107556]: pgmap v425: 99 pgs: 99 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 770 KiB/s, 12 objects/s recovering 2024-09-13T14:29:39.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/599918873' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:38 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:29:39.295 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:39.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:29:39.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:39.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:39.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:39.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:39.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.610 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:39.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:39.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:39.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:39.789 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:39.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:39.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:39.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:39.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:40.043 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:40.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:40.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:40.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:29:40.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[103001]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:29:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:29:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:39 smithi017 ceph-mon[107556]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:29:40.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:39 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:29:40.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:29:40.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:39 smithi189 ceph-mon[90992]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:29:40.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:40.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:40.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:40.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:40.457 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:40.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:40.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:40.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:40.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:40.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:40.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:40.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:40.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:40.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:41.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:41.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:41.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:41.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:40 smithi017 ceph-mon[103001]: pgmap v427: 99 pgs: 99 active+clean; 915 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 692 KiB/s, 10 objects/s recovering 2024-09-13T14:29:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:40 smithi017 ceph-mon[107556]: pgmap v427: 99 pgs: 99 active+clean; 915 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 692 KiB/s, 10 objects/s recovering 2024-09-13T14:29:41.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:40 smithi189 ceph-mon[90992]: pgmap v427: 99 pgs: 99 active+clean; 915 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 692 KiB/s, 10 objects/s recovering 2024-09-13T14:29:41.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:41.315 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:41.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:41.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:41.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:41.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:41.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:41.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:41.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:41.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:41.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:41.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:41.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:42.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:42.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:42.246 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:42.278 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:42.308 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:42.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:42.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:42.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:42.545 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:42.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:42.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:42.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:42.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:42.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:42.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:42.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:29:42.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:43.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:43.142 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:43.173 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[103001]: pgmap v428: 99 pgs: 99 active+clean; 848 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 685 KiB/s, 10 objects/s recovering 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[107556]: pgmap v428: 99 pgs: 99 active+clean; 848 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 685 KiB/s, 10 objects/s recovering 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:43.253 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:42 smithi189 ceph-mon[90992]: pgmap v428: 99 pgs: 99 active+clean; 848 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 685 KiB/s, 10 objects/s recovering 2024-09-13T14:29:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:43.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:43.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:43.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:43.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:43.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:43.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:43.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:43.696 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:43.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:43.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:43.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:43.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:43.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:44.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:44.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:44.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:44.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:44.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:44.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:44.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:44.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:44.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:44.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:44.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:44.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:44.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:44.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:44.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:44.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:44.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:45.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:45.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:45.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:45.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:45.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:45.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:45.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:45.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:45.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:45.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:45 smithi017 ceph-mon[103001]: pgmap v429: 99 pgs: 99 active+clean; 681 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:29:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:45 smithi017 ceph-mon[107556]: pgmap v429: 99 pgs: 99 active+clean; 681 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:29:45.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:45 smithi189 ceph-mon[90992]: pgmap v429: 99 pgs: 99 active+clean; 681 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:29:45.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:45.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:45.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:45.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:45.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:46.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:46.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:46.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:46.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:46.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:46.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:46.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:46.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:46.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:46.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:46.588 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:46.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:46.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:46.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:46.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:46.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:46.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:46.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:47.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:47.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:47.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:47.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:47.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:47.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:47] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T14:29:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:47 smithi017 ceph-mon[103001]: pgmap v430: 99 pgs: 99 active+clean; 623 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:29:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:47 smithi017 ceph-mon[107556]: pgmap v430: 99 pgs: 99 active+clean; 623 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:29:47.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:47.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:47.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:47 smithi189 ceph-mon[90992]: pgmap v430: 99 pgs: 99 active+clean; 623 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:29:47.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:47.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:47.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:47.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:47.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:48.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:48.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:48.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:48.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:48.276 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:29:48.276 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:29:48.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:48.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:48.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:48.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:48.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:48.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:48.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:48.859 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:48.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:48.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:48.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:49.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:49.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:49.154 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:29:49.160 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:29:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:49.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:49.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:49.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:49.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:49.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:49 smithi017 ceph-mon[103001]: pgmap v431: 99 pgs: 99 active+clean; 489 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:29:49.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1398973378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:49.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:49 smithi017 ceph-mon[107556]: pgmap v431: 99 pgs: 99 active+clean; 489 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:29:49.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1398973378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:49.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:49.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:49.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:49.561 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:29:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:29:49.595 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:29:49.596 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:29:49.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:49.641 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:29:49.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:29:49.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:49.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:49.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:49 smithi189 ceph-mon[90992]: pgmap v431: 99 pgs: 99 active+clean; 489 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:29:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1398973378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:29:49.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:49.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:49.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:49.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:50.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:50.111 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":431,"stamp":"2024-09-13T14:29:47.814460+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":512825317,"num_objects":7781,"num_object_clones":0,"num_object_copies":23343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7781,"num_whiteouts":0,"num_read":6305,"num_read_kb":13771,"num_write":153065,"num_write_kb":3239596,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20482,"num_bytes_recovered":1342702112,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":280340,"ondisk_log_size":280340,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3747152,"kb_used_data":1875536,"kb_used_omap":35,"kb_used_meta":1871260,"kb_avail":746214064,"statfs":{"total":767960285184,"available":764123201536,"internally_reserved":0,"allocated":1920548864,"data_stored":1913520583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1916171257},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-629276672,"num_objects":-9602,"num_object_clones":0,"num_object_copies":-28806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002806"},"pg_stats":[{"pgid":"5.15","version":"162'13499","reported_seq":16109,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.880442+0000","last_change":"2024-09-13T14:29:09.443791+0000","last_active":"2024-09-13T14:29:42.880442+0000","last_peered":"2024-09-13T14:29:42.880442+0000","last_clean":"2024-09-13T14:29:42.880442+0000","last_became_active":"2024-09-13T14:29:09.409873+0000","last_became_peered":"2024-09-13T14:29:09.409873+0000","last_unstale":"2024-09-13T14:29:42.880442+0000","last_undegraded":"2024-09-13T14:29:42.880442+0000","last_fullsized":"2024-09-13T14:29:42.880442+0000","mapping_epoch":146,"log_start":"109'1900","ondisk_log_start":"109'1900","created":119,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'10832","last_scrub_stamp":"2024-09-13T14:27:56.789292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.789292+0000","objects_scrubbed":2994,"log_size":11599,"log_dups_size":0,"ondisk_log_size":11599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:30.346914+0000","scrub_duration":947,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20709376,"num_objects":316,"num_object_clones":0,"num_object_copies":948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2678,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79626240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965179+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:29:38.965179+0000","last_peered":"2024-09-13T14:29:38.965179+0000","last_clean":"2024-09-13T14:29:38.965179+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:29:38.965179+0000","last_undegraded":"2024-09-13T14:29:38.965179+0000","last_fullsized":"2024-09-13T14:29:38.965179+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":128,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927281+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:29:29.927281+0000","last_peered":"2024-09-13T14:29:29.927281+0000","last_clean":"2024-09-13T14:29:29.927281+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:29:29.927281+0000","last_undegraded":"2024-09-13T14:29:29.927281+0000","last_fullsized":"2024-09-13T14:29:29.927281+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.14","version":"162'13467","reported_seq":15813,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:43.659491+0000","last_change":"2024-09-13T14:29:17.475964+0000","last_active":"2024-09-13T14:29:43.659491+0000","last_peered":"2024-09-13T14:29:43.659491+0000","last_clean":"2024-09-13T14:29:43.659491+0000","last_became_active":"2024-09-13T14:29:17.450196+0000","last_became_peered":"2024-09-13T14:29:17.450196+0000","last_unstale":"2024-09-13T14:29:43.659491+0000","last_undegraded":"2024-09-13T14:29:43.659491+0000","last_fullsized":"2024-09-13T14:29:43.659491+0000","mapping_epoch":151,"log_start":"109'2000","ondisk_log_start":"109'2000","created":119,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":5,"last_scrub":"122'10895","last_scrub_stamp":"2024-09-13T14:28:00.826750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:28:00.826750+0000","objects_scrubbed":2834,"log_size":11467,"log_dups_size":0,"ondisk_log_size":11467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:41.735260+0000","scrub_duration":0.93828766600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16384000,"num_objects":250,"num_object_clones":0,"num_object_copies":750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2584,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1049,"num_bytes_recovered":68681728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927268+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:29:29.927268+0000","last_peered":"2024-09-13T14:29:29.927268+0000","last_clean":"2024-09-13T14:29:29.927268+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:29:29.927268+0000","last_undegraded":"2024-09-13T14:29:29.927268+0000","last_fullsized":"2024-09-13T14:29:29.927268+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":186,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972958+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:29:38.972958+0000","last_peered":"2024-09-13T14:29:38.972958+0000","last_clean":"2024-09-13T14:29:38.972958+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:29:38.972958+0000","last_undegraded":"2024-09-13T14:29:38.972958+0000","last_fullsized":"2024-09-13T14:29:38.972958+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.17","version":"162'13596","reported_seq":16683,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:43.812373+0000","last_change":"2024-09-13T14:29:21.529723+0000","last_active":"2024-09-13T14:29:43.812373+0000","last_peered":"2024-09-13T14:29:43.812373+0000","last_clean":"2024-09-13T14:29:43.812373+0000","last_became_active":"2024-09-13T14:29:21.487371+0000","last_became_peered":"2024-09-13T14:29:21.487371+0000","last_unstale":"2024-09-13T14:29:43.812373+0000","last_undegraded":"2024-09-13T14:29:43.812373+0000","last_fullsized":"2024-09-13T14:29:43.812373+0000","mapping_epoch":155,"log_start":"109'1967","ondisk_log_start":"109'1967","created":119,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'10878","last_scrub_stamp":"2024-09-13T14:27:56.784923+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.784923+0000","objects_scrubbed":3032,"log_size":11629,"log_dups_size":0,"ondisk_log_size":11629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:33:36.929287+0000","scrub_duration":1.3025174100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267584,"num_objects":294,"num_object_clones":0,"num_object_copies":882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2738,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":950,"num_bytes_recovered":62259200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":187,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978123+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:29:38.978123+0000","last_peered":"2024-09-13T14:29:38.978123+0000","last_clean":"2024-09-13T14:29:38.978123+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:29:38.978123+0000","last_undegraded":"2024-09-13T14:29:38.978123+0000","last_fullsized":"2024-09-13T14:29:38.978123+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927773+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:29:29.927773+0000","last_peered":"2024-09-13T14:29:29.927773+0000","last_clean":"2024-09-13T14:29:29.927773+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:29:29.927773+0000","last_undegraded":"2024-09-13T14:29:29.927773+0000","last_fullsized":"2024-09-13T14:29:29.927773+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.16","version":"162'13413","reported_seq":16676,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.896869+0000","last_change":"2024-09-13T14:29:21.513548+0000","last_active":"2024-09-13T14:29:42.896869+0000","last_peered":"2024-09-13T14:29:42.896869+0000","last_clean":"2024-09-13T14:29:42.896869+0000","last_became_active":"2024-09-13T14:29:21.487383+0000","last_became_peered":"2024-09-13T14:29:21.487383+0000","last_unstale":"2024-09-13T14:29:42.896869+0000","last_undegraded":"2024-09-13T14:29:42.896869+0000","last_fullsized":"2024-09-13T14:29:42.896869+0000","mapping_epoch":155,"log_start":"108'600","ondisk_log_start":"108'600","created":119,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"121'10767","last_scrub_stamp":"2024-09-13T14:27:57.559369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:57.559369+0000","objects_scrubbed":2944,"log_size":12813,"log_dups_size":0,"ondisk_log_size":12813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:30:11.000969+0000","scrub_duration":803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2655,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1024,"num_bytes_recovered":67108864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965770+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:29:38.965770+0000","last_peered":"2024-09-13T14:29:38.965770+0000","last_clean":"2024-09-13T14:29:38.965770+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:29:38.965770+0000","last_undegraded":"2024-09-13T14:29:38.965770+0000","last_fullsized":"2024-09-13T14:29:38.965770+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":134,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924194+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:29:29.924194+0000","last_peered":"2024-09-13T14:29:29.924194+0000","last_clean":"2024-09-13T14:29:29.924194+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:29:29.924194+0000","last_undegraded":"2024-09-13T14:29:29.924194+0000","last_fullsized":"2024-09-13T14:29:29.924194+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"162'13371","reported_seq":13572,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.831492+0000","last_change":"2024-09-13T14:27:59.744122+0000","last_active":"2024-09-13T14:29:42.831492+0000","last_peered":"2024-09-13T14:29:42.831492+0000","last_clean":"2024-09-13T14:29:42.831492+0000","last_became_active":"2024-09-13T14:27:53.311848+0000","last_became_peered":"2024-09-13T14:27:53.311848+0000","last_unstale":"2024-09-13T14:29:42.831492+0000","last_undegraded":"2024-09-13T14:29:42.831492+0000","last_fullsized":"2024-09-13T14:29:42.831492+0000","mapping_epoch":104,"log_start":"109'2000","ondisk_log_start":"109'2000","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"121'10819","last_scrub_stamp":"2024-09-13T14:27:59.743966+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:59.743966+0000","objects_scrubbed":2851,"log_size":11371,"log_dups_size":0,"ondisk_log_size":11371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:58:38.271483+0000","scrub_duration":0.90100438699999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":127,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927569+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:29:29.927569+0000","last_peered":"2024-09-13T14:29:29.927569+0000","last_clean":"2024-09-13T14:29:29.927569+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:29:29.927569+0000","last_undegraded":"2024-09-13T14:29:29.927569+0000","last_fullsized":"2024-09-13T14:29:29.927569+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926682+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:29:29.926682+0000","last_peered":"2024-09-13T14:29:29.926682+0000","last_clean":"2024-09-13T14:29:29.926682+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:29:29.926682+0000","last_undegraded":"2024-09-13T14:29:29.926682+0000","last_fullsized":"2024-09-13T14:29:29.926682+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"162'13592","reported_seq":13777,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:45.837346+0000","last_change":"2024-09-13T14:27:55.292367+0000","last_active":"2024-09-13T14:29:45.837346+0000","last_peered":"2024-09-13T14:29:45.837346+0000","last_clean":"2024-09-13T14:29:45.837346+0000","last_became_active":"2024-09-13T14:27:53.285311+0000","last_became_peered":"2024-09-13T14:27:53.285311+0000","last_unstale":"2024-09-13T14:29:45.837346+0000","last_undegraded":"2024-09-13T14:29:45.837346+0000","last_fullsized":"2024-09-13T14:29:45.837346+0000","mapping_epoch":104,"log_start":"108'700","ondisk_log_start":"108'700","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10778","last_scrub_stamp":"2024-09-13T14:27:55.292058+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:55.292058+0000","objects_scrubbed":3041,"log_size":12892,"log_dups_size":0,"ondisk_log_size":12892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:06.504074+0000","scrub_duration":1447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13762560,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2831,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":128,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927666+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:29:29.927666+0000","last_peered":"2024-09-13T14:29:29.927666+0000","last_clean":"2024-09-13T14:29:29.927666+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:29:29.927666+0000","last_undegraded":"2024-09-13T14:29:29.927666+0000","last_fullsized":"2024-09-13T14:29:29.927666+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956315+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:29:29.956315+0000","last_peered":"2024-09-13T14:29:29.956315+0000","last_clean":"2024-09-13T14:29:29.956315+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:29:29.956315+0000","last_undegraded":"2024-09-13T14:29:29.956315+0000","last_fullsized":"2024-09-13T14:29:29.956315+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"162'16745","reported_seq":21818,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:45.925133+0000","last_change":"2024-09-13T14:29:11.458559+0000","last_active":"2024-09-13T14:29:45.925133+0000","last_peered":"2024-09-13T14:29:45.925133+0000","last_clean":"2024-09-13T14:29:45.925133+0000","last_became_active":"2024-09-13T14:29:11.422111+0000","last_became_peered":"2024-09-13T14:29:11.422111+0000","last_unstale":"2024-09-13T14:29:45.925133+0000","last_undegraded":"2024-09-13T14:29:45.925133+0000","last_fullsized":"2024-09-13T14:29:45.925133+0000","mapping_epoch":148,"log_start":"112'6700","ondisk_log_start":"112'6700","created":104,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:49:53.869886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27328512,"num_objects":417,"num_object_clones":0,"num_object_copies":1251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16745,"num_write_kb":549184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2460,"num_bytes_recovered":161218560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":110,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927499+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:29:29.927499+0000","last_peered":"2024-09-13T14:29:29.927499+0000","last_clean":"2024-09-13T14:29:29.927499+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:29:29.927499+0000","last_undegraded":"2024-09-13T14:29:29.927499+0000","last_fullsized":"2024-09-13T14:29:29.927499+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924377+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:29:29.924377+0000","last_peered":"2024-09-13T14:29:29.924377+0000","last_clean":"2024-09-13T14:29:29.924377+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:29:29.924377+0000","last_undegraded":"2024-09-13T14:29:29.924377+0000","last_fullsized":"2024-09-13T14:29:29.924377+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"162'16217","reported_seq":16357,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:45.908216+0000","last_change":"2024-09-13T14:25:29.103217+0000","last_active":"2024-09-13T14:29:45.908216+0000","last_peered":"2024-09-13T14:29:45.908216+0000","last_clean":"2024-09-13T14:29:45.908216+0000","last_became_active":"2024-09-13T14:25:29.102866+0000","last_became_peered":"2024-09-13T14:25:29.102866+0000","last_unstale":"2024-09-13T14:29:45.908216+0000","last_undegraded":"2024-09-13T14:29:45.908216+0000","last_fullsized":"2024-09-13T14:29:45.908216+0000","mapping_epoch":104,"log_start":"111'6200","ondisk_log_start":"111'6200","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:37.178819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16217,"num_write_kb":531904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927117+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:29:29.927117+0000","last_peered":"2024-09-13T14:29:29.927117+0000","last_clean":"2024-09-13T14:29:29.927117+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:29:29.927117+0000","last_undegraded":"2024-09-13T14:29:29.927117+0000","last_fullsized":"2024-09-13T14:29:29.927117+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":187,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972620+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:29:38.972620+0000","last_peered":"2024-09-13T14:29:38.972620+0000","last_clean":"2024-09-13T14:29:38.972620+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:29:38.972620+0000","last_undegraded":"2024-09-13T14:29:38.972620+0000","last_fullsized":"2024-09-13T14:29:38.972620+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"162'16593","reported_seq":16734,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:45.900774+0000","last_change":"2024-09-13T14:25:29.106845+0000","last_active":"2024-09-13T14:29:45.900774+0000","last_peered":"2024-09-13T14:29:45.900774+0000","last_clean":"2024-09-13T14:29:45.900774+0000","last_became_active":"2024-09-13T14:25:29.106506+0000","last_became_peered":"2024-09-13T14:25:29.106506+0000","last_unstale":"2024-09-13T14:29:45.900774+0000","last_undegraded":"2024-09-13T14:29:45.900774+0000","last_fullsized":"2024-09-13T14:29:45.900774+0000","mapping_epoch":104,"log_start":"111'6500","ondisk_log_start":"111'6500","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:28.391210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27787288,"num_objects":425,"num_object_clones":0,"num_object_copies":1275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":425,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16593,"num_write_kb":544513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":188,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965627+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:29:38.965627+0000","last_peered":"2024-09-13T14:29:38.965627+0000","last_clean":"2024-09-13T14:29:38.965627+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:29:38.965627+0000","last_undegraded":"2024-09-13T14:29:38.965627+0000","last_fullsized":"2024-09-13T14:29:38.965627+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956482+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:29:29.956482+0000","last_peered":"2024-09-13T14:29:29.956482+0000","last_clean":"2024-09-13T14:29:29.956482+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:29:29.956482+0000","last_undegraded":"2024-09-13T14:29:29.956482+0000","last_fullsized":"2024-09-13T14:29:29.956482+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"162'16402","reported_seq":16542,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:47.189186+0000","last_change":"2024-09-13T14:25:29.103072+0000","last_active":"2024-09-13T14:29:47.189186+0000","last_peered":"2024-09-13T14:29:47.189186+0000","last_clean":"2024-09-13T14:29:47.189186+0000","last_became_active":"2024-09-13T14:25:29.102737+0000","last_became_peered":"2024-09-13T14:25:29.102737+0000","last_unstale":"2024-09-13T14:29:47.189186+0000","last_undegraded":"2024-09-13T14:29:47.189186+0000","last_fullsized":"2024-09-13T14:29:47.189186+0000","mapping_epoch":104,"log_start":"116'9710","ondisk_log_start":"116'9710","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:33:32.031601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19529728,"num_objects":298,"num_object_clones":0,"num_object_copies":894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16402,"num_write_kb":534400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924284+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:29:29.924284+0000","last_peered":"2024-09-13T14:29:29.924284+0000","last_clean":"2024-09-13T14:29:29.924284+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:29:29.924284+0000","last_undegraded":"2024-09-13T14:29:29.924284+0000","last_fullsized":"2024-09-13T14:29:29.924284+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956763+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:29:29.956763+0000","last_peered":"2024-09-13T14:29:29.956763+0000","last_clean":"2024-09-13T14:29:29.956763+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:29:29.956763+0000","last_undegraded":"2024-09-13T14:29:29.956763+0000","last_fullsized":"2024-09-13T14:29:29.956763+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"162'16094","reported_seq":24545,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:43.840719+0000","last_change":"2024-09-13T14:28:47.933962+0000","last_active":"2024-09-13T14:29:43.840719+0000","last_peered":"2024-09-13T14:29:43.840719+0000","last_clean":"2024-09-13T14:29:43.840719+0000","last_became_active":"2024-09-13T14:28:47.884983+0000","last_became_peered":"2024-09-13T14:28:47.884983+0000","last_unstale":"2024-09-13T14:29:43.840719+0000","last_undegraded":"2024-09-13T14:29:43.840719+0000","last_fullsized":"2024-09-13T14:29:43.840719+0000","mapping_epoch":135,"log_start":"114'8784","ondisk_log_start":"114'8784","created":104,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":7310,"log_dups_size":0,"ondisk_log_size":7310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:26:57.097810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16094,"num_write_kb":532992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4168,"num_bytes_recovered":273154048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":127,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956682+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:29:29.956682+0000","last_peered":"2024-09-13T14:29:29.956682+0000","last_clean":"2024-09-13T14:29:29.956682+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:29:29.956682+0000","last_undegraded":"2024-09-13T14:29:29.956682+0000","last_fullsized":"2024-09-13T14:29:29.956682+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956834+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:29:29.956834+0000","last_peered":"2024-09-13T14:29:29.956834+0000","last_clean":"2024-09-13T14:29:29.956834+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:29:29.956834+0000","last_undegraded":"2024-09-13T14:29:29.956834+0000","last_fullsized":"2024-09-13T14:29:29.956834+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"162'16087","reported_seq":16153,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.953447+0000","last_change":"2024-09-13T14:25:29.105317+0000","last_active":"2024-09-13T14:29:42.953447+0000","last_peered":"2024-09-13T14:29:42.953447+0000","last_clean":"2024-09-13T14:29:42.953447+0000","last_became_active":"2024-09-13T14:25:29.104986+0000","last_became_peered":"2024-09-13T14:25:29.104986+0000","last_unstale":"2024-09-13T14:29:42.953447+0000","last_undegraded":"2024-09-13T14:29:42.953447+0000","last_fullsized":"2024-09-13T14:29:42.953447+0000","mapping_epoch":104,"log_start":"115'9557","ondisk_log_start":"115'9557","created":104,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:25:28.084709+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:21.431572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35979264,"num_objects":549,"num_object_clones":0,"num_object_copies":1647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16087,"num_write_kb":532352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965422+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:29:38.965422+0000","last_peered":"2024-09-13T14:29:38.965422+0000","last_clean":"2024-09-13T14:29:38.965422+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:29:38.965422+0000","last_undegraded":"2024-09-13T14:29:38.965422+0000","last_fullsized":"2024-09-13T14:29:38.965422+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":177,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.982605+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:29:38.982605+0000","last_peered":"2024-09-13T14:29:38.982605+0000","last_clean":"2024-09-13T14:29:38.982605+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:29:38.982605+0000","last_undegraded":"2024-09-13T14:29:38.982605+0000","last_fullsized":"2024-09-13T14:29:38.982605+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"162'13425","reported_seq":13630,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.891875+0000","last_change":"2024-09-13T14:27:58.740430+0000","last_active":"2024-09-13T14:29:42.891875+0000","last_peered":"2024-09-13T14:29:42.891875+0000","last_clean":"2024-09-13T14:29:42.891875+0000","last_became_active":"2024-09-13T14:27:53.337621+0000","last_became_peered":"2024-09-13T14:27:53.337621+0000","last_unstale":"2024-09-13T14:29:42.891875+0000","last_undegraded":"2024-09-13T14:29:42.891875+0000","last_fullsized":"2024-09-13T14:29:42.891875+0000","mapping_epoch":104,"log_start":"109'2000","ondisk_log_start":"109'2000","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"121'10811","last_scrub_stamp":"2024-09-13T14:27:58.740290+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:58.740290+0000","objects_scrubbed":2893,"log_size":11425,"log_dups_size":0,"ondisk_log_size":11425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:41.146299+0000","scrub_duration":0.879492198,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17563648,"num_objects":268,"num_object_clones":0,"num_object_copies":804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2625,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":243,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926955+0000","last_change":"2024-09-13T14:21:39.297797+0000","last_active":"2024-09-13T14:29:29.926955+0000","last_peered":"2024-09-13T14:29:29.926955+0000","last_clean":"2024-09-13T14:29:29.926955+0000","last_became_active":"2024-09-13T14:21:39.296625+0000","last_became_peered":"2024-09-13T14:21:39.296625+0000","last_unstale":"2024-09-13T14:29:29.926955+0000","last_undegraded":"2024-09-13T14:29:29.926955+0000","last_fullsized":"2024-09-13T14:29:29.926955+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:06:03.833277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":727,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926920+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:29:29.926920+0000","last_peered":"2024-09-13T14:29:29.926920+0000","last_clean":"2024-09-13T14:29:29.926920+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:29:29.926920+0000","last_undegraded":"2024-09-13T14:29:29.926920+0000","last_fullsized":"2024-09-13T14:29:29.926920+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.966218+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:29:38.966218+0000","last_peered":"2024-09-13T14:29:38.966218+0000","last_clean":"2024-09-13T14:29:38.966218+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:29:38.966218+0000","last_undegraded":"2024-09-13T14:29:38.966218+0000","last_fullsized":"2024-09-13T14:29:38.966218+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924506+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:29:29.924506+0000","last_peered":"2024-09-13T14:29:29.924506+0000","last_clean":"2024-09-13T14:29:29.924506+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:29:29.924506+0000","last_undegraded":"2024-09-13T14:29:29.924506+0000","last_fullsized":"2024-09-13T14:29:29.924506+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"162'13531","reported_seq":13737,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.809896+0000","last_change":"2024-09-13T14:27:55.833775+0000","last_active":"2024-09-13T14:29:42.809896+0000","last_peered":"2024-09-13T14:29:42.809896+0000","last_clean":"2024-09-13T14:29:42.809896+0000","last_became_active":"2024-09-13T14:27:53.316126+0000","last_became_peered":"2024-09-13T14:27:53.316126+0000","last_unstale":"2024-09-13T14:29:42.809896+0000","last_undegraded":"2024-09-13T14:29:42.809896+0000","last_fullsized":"2024-09-13T14:29:42.809896+0000","mapping_epoch":104,"log_start":"108'700","ondisk_log_start":"108'700","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10852","last_scrub_stamp":"2024-09-13T14:27:55.833721+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:55.833721+0000","objects_scrubbed":2997,"log_size":12831,"log_dups_size":0,"ondisk_log_size":12831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:37.251899+0000","scrub_duration":1.3079477129999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19529728,"num_objects":298,"num_object_clones":0,"num_object_copies":894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2699,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":535,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926869+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:29:29.926869+0000","last_peered":"2024-09-13T14:29:29.926869+0000","last_clean":"2024-09-13T14:29:29.926869+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:29:29.926869+0000","last_undegraded":"2024-09-13T14:29:29.926869+0000","last_fullsized":"2024-09-13T14:29:29.926869+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":186,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.971222+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:29:38.971222+0000","last_peered":"2024-09-13T14:29:38.971222+0000","last_clean":"2024-09-13T14:29:38.971222+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:29:38.971222+0000","last_undegraded":"2024-09-13T14:29:38.971222+0000","last_fullsized":"2024-09-13T14:29:38.971222+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":198,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978287+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:29:38.978287+0000","last_peered":"2024-09-13T14:29:38.978287+0000","last_clean":"2024-09-13T14:29:38.978287+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:29:38.978287+0000","last_undegraded":"2024-09-13T14:29:38.978287+0000","last_fullsized":"2024-09-13T14:29:38.978287+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"162'13558","reported_seq":13743,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:45.935414+0000","last_change":"2024-09-13T14:27:55.878332+0000","last_active":"2024-09-13T14:29:45.935414+0000","last_peered":"2024-09-13T14:29:45.935414+0000","last_clean":"2024-09-13T14:29:45.935414+0000","last_became_active":"2024-09-13T14:27:53.298430+0000","last_became_peered":"2024-09-13T14:27:53.298430+0000","last_unstale":"2024-09-13T14:29:45.935414+0000","last_undegraded":"2024-09-13T14:29:45.935414+0000","last_fullsized":"2024-09-13T14:29:45.935414+0000","mapping_epoch":104,"log_start":"108'600","ondisk_log_start":"108'600","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10732","last_scrub_stamp":"2024-09-13T14:27:55.878215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:55.878215+0000","objects_scrubbed":3022,"log_size":12958,"log_dups_size":0,"ondisk_log_size":12958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:09.324342+0000","scrub_duration":1055,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12124160,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2837,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":616,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.968822+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:29:38.968822+0000","last_peered":"2024-09-13T14:29:38.968822+0000","last_clean":"2024-09-13T14:29:38.968822+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:29:38.968822+0000","last_undegraded":"2024-09-13T14:29:38.968822+0000","last_fullsized":"2024-09-13T14:29:38.968822+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.986421+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:29:38.986421+0000","last_peered":"2024-09-13T14:29:38.986421+0000","last_clean":"2024-09-13T14:29:38.986421+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:29:38.986421+0000","last_undegraded":"2024-09-13T14:29:38.986421+0000","last_fullsized":"2024-09-13T14:29:38.986421+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.968441+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:29:38.968441+0000","last_peered":"2024-09-13T14:29:38.968441+0000","last_clean":"2024-09-13T14:29:38.968441+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:29:38.968441+0000","last_undegraded":"2024-09-13T14:29:38.968441+0000","last_fullsized":"2024-09-13T14:29:38.968441+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.5","version":"162'13637","reported_seq":13823,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:47.340634+0000","last_change":"2024-09-13T14:27:55.007486+0000","last_active":"2024-09-13T14:29:47.340634+0000","last_peered":"2024-09-13T14:29:47.340634+0000","last_clean":"2024-09-13T14:29:47.340634+0000","last_became_active":"2024-09-13T14:27:53.307251+0000","last_became_peered":"2024-09-13T14:27:53.307251+0000","last_unstale":"2024-09-13T14:29:47.340634+0000","last_undegraded":"2024-09-13T14:29:47.340634+0000","last_fullsized":"2024-09-13T14:29:47.340634+0000","mapping_epoch":104,"log_start":"109'1900","ondisk_log_start":"109'1900","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10780","last_scrub_stamp":"2024-09-13T14:27:55.006982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:55.006982+0000","objects_scrubbed":3062,"log_size":11737,"log_dups_size":0,"ondisk_log_size":11737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:02.021879+0000","scrub_duration":1091,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12255232,"num_objects":187,"num_object_clones":0,"num_object_copies":561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2875,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":776,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926917+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:29:29.926917+0000","last_peered":"2024-09-13T14:29:29.926917+0000","last_clean":"2024-09-13T14:29:29.926917+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:29:29.926917+0000","last_undegraded":"2024-09-13T14:29:29.926917+0000","last_fullsized":"2024-09-13T14:29:29.926917+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956445+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:29:29.956445+0000","last_peered":"2024-09-13T14:29:29.956445+0000","last_clean":"2024-09-13T14:29:29.956445+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:29:29.956445+0000","last_undegraded":"2024-09-13T14:29:29.956445+0000","last_fullsized":"2024-09-13T14:29:29.956445+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":188,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978539+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:29:38.978539+0000","last_peered":"2024-09-13T14:29:38.978539+0000","last_clean":"2024-09-13T14:29:38.978539+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:29:38.978539+0000","last_undegraded":"2024-09-13T14:29:38.978539+0000","last_fullsized":"2024-09-13T14:29:38.978539+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"162'13656","reported_seq":13842,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:47.384470+0000","last_change":"2024-09-13T14:27:56.222330+0000","last_active":"2024-09-13T14:29:47.384470+0000","last_peered":"2024-09-13T14:29:47.384470+0000","last_clean":"2024-09-13T14:29:47.384470+0000","last_became_active":"2024-09-13T14:27:53.309179+0000","last_became_peered":"2024-09-13T14:27:53.309179+0000","last_unstale":"2024-09-13T14:29:47.384470+0000","last_undegraded":"2024-09-13T14:29:47.384470+0000","last_fullsized":"2024-09-13T14:29:47.384470+0000","mapping_epoch":104,"log_start":"109'1900","ondisk_log_start":"109'1900","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10830","last_scrub_stamp":"2024-09-13T14:27:56.222198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.222198+0000","objects_scrubbed":2985,"log_size":11756,"log_dups_size":0,"ondisk_log_size":11756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:37.279661+0000","scrub_duration":1339,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9371648,"num_objects":143,"num_object_clones":0,"num_object_copies":429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2842,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927355+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:29:29.927355+0000","last_peered":"2024-09-13T14:29:29.927355+0000","last_clean":"2024-09-13T14:29:29.927355+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:29:29.927355+0000","last_undegraded":"2024-09-13T14:29:29.927355+0000","last_fullsized":"2024-09-13T14:29:29.927355+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":787,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.979201+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:29:38.979201+0000","last_peered":"2024-09-13T14:29:38.979201+0000","last_clean":"2024-09-13T14:29:38.979201+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:29:38.979201+0000","last_undegraded":"2024-09-13T14:29:38.979201+0000","last_fullsized":"2024-09-13T14:29:38.979201+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":166,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978319+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:29:38.978319+0000","last_peered":"2024-09-13T14:29:38.978319+0000","last_clean":"2024-09-13T14:29:38.978319+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:29:38.978319+0000","last_undegraded":"2024-09-13T14:29:38.978319+0000","last_fullsized":"2024-09-13T14:29:38.978319+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"162'13524","reported_seq":13727,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.804396+0000","last_change":"2024-09-13T14:28:00.750352+0000","last_active":"2024-09-13T14:29:42.804396+0000","last_peered":"2024-09-13T14:29:42.804396+0000","last_clean":"2024-09-13T14:29:42.804396+0000","last_became_active":"2024-09-13T14:27:53.333014+0000","last_became_peered":"2024-09-13T14:27:53.333014+0000","last_unstale":"2024-09-13T14:29:42.804396+0000","last_undegraded":"2024-09-13T14:29:42.804396+0000","last_fullsized":"2024-09-13T14:29:42.804396+0000","mapping_epoch":104,"log_start":"108'1000","ondisk_log_start":"108'1000","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"122'10941","last_scrub_stamp":"2024-09-13T14:28:00.750212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:28:00.750212+0000","objects_scrubbed":2894,"log_size":12524,"log_dups_size":0,"ondisk_log_size":12524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:37.170041+0000","scrub_duration":0.92449736699999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2599,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":177,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.979069+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:29:38.979069+0000","last_peered":"2024-09-13T14:29:38.979069+0000","last_clean":"2024-09-13T14:29:38.979069+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:29:38.979069+0000","last_undegraded":"2024-09-13T14:29:38.979069+0000","last_fullsized":"2024-09-13T14:29:38.979069+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":472,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927542+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:29:29.927542+0000","last_peered":"2024-09-13T14:29:29.927542+0000","last_clean":"2024-09-13T14:29:29.927542+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:29:29.927542+0000","last_undegraded":"2024-09-13T14:29:29.927542+0000","last_fullsized":"2024-09-13T14:29:29.927542+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":176,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972256+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:29:38.972256+0000","last_peered":"2024-09-13T14:29:38.972256+0000","last_clean":"2024-09-13T14:29:38.972256+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:29:38.972256+0000","last_undegraded":"2024-09-13T14:29:38.972256+0000","last_fullsized":"2024-09-13T14:29:38.972256+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"162'13481","reported_seq":13686,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.879841+0000","last_change":"2024-09-13T14:27:56.753622+0000","last_active":"2024-09-13T14:29:42.879841+0000","last_peered":"2024-09-13T14:29:42.879841+0000","last_clean":"2024-09-13T14:29:42.879841+0000","last_became_active":"2024-09-13T14:27:53.302280+0000","last_became_peered":"2024-09-13T14:27:53.302280+0000","last_unstale":"2024-09-13T14:29:42.879841+0000","last_undegraded":"2024-09-13T14:29:42.879841+0000","last_fullsized":"2024-09-13T14:29:42.879841+0000","mapping_epoch":104,"log_start":"109'3300","ondisk_log_start":"109'3300","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10793","last_scrub_stamp":"2024-09-13T14:27:56.753194+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.753194+0000","objects_scrubbed":2979,"log_size":10181,"log_dups_size":0,"ondisk_log_size":10181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:04.486002+0000","scrub_duration":0.82935717099999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18284544,"num_objects":279,"num_object_clones":0,"num_object_copies":837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2700,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":188,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.966353+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:29:38.966353+0000","last_peered":"2024-09-13T14:29:38.966353+0000","last_clean":"2024-09-13T14:29:38.966353+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:29:38.966353+0000","last_undegraded":"2024-09-13T14:29:38.966353+0000","last_fullsized":"2024-09-13T14:29:38.966353+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":521,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.979106+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:29:38.979106+0000","last_peered":"2024-09-13T14:29:38.979106+0000","last_clean":"2024-09-13T14:29:38.979106+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:29:38.979106+0000","last_undegraded":"2024-09-13T14:29:38.979106+0000","last_fullsized":"2024-09-13T14:29:38.979106+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965245+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:29:38.965245+0000","last_peered":"2024-09-13T14:29:38.965245+0000","last_clean":"2024-09-13T14:29:38.965245+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:29:38.965245+0000","last_undegraded":"2024-09-13T14:29:38.965245+0000","last_fullsized":"2024-09-13T14:29:38.965245+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"162'13622","reported_seq":13809,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:43.734241+0000","last_change":"2024-09-13T14:27:55.284217+0000","last_active":"2024-09-13T14:29:43.734241+0000","last_peered":"2024-09-13T14:29:43.734241+0000","last_clean":"2024-09-13T14:29:43.734241+0000","last_became_active":"2024-09-13T14:27:53.308093+0000","last_became_peered":"2024-09-13T14:27:53.308093+0000","last_unstale":"2024-09-13T14:29:43.734241+0000","last_undegraded":"2024-09-13T14:29:43.734241+0000","last_fullsized":"2024-09-13T14:29:43.734241+0000","mapping_epoch":104,"log_start":"109'2175","ondisk_log_start":"109'2175","created":104,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'10855","last_scrub_stamp":"2024-09-13T14:27:55.284101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:55.284101+0000","objects_scrubbed":3071,"log_size":11447,"log_dups_size":0,"ondisk_log_size":11447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:15.399024+0000","scrub_duration":1149,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2776,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.977981+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:29:38.977981+0000","last_peered":"2024-09-13T14:29:38.977981+0000","last_clean":"2024-09-13T14:29:38.977981+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:29:38.977981+0000","last_undegraded":"2024-09-13T14:29:38.977981+0000","last_fullsized":"2024-09-13T14:29:38.977981+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":580,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924573+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:29:29.924573+0000","last_peered":"2024-09-13T14:29:29.924573+0000","last_clean":"2024-09-13T14:29:29.924573+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:29:29.924573+0000","last_undegraded":"2024-09-13T14:29:29.924573+0000","last_fullsized":"2024-09-13T14:29:29.924573+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":109,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956290+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:29:29.956290+0000","last_peered":"2024-09-13T14:29:29.956290+0000","last_clean":"2024-09-13T14:29:29.956290+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:29:29.956290+0000","last_undegraded":"2024-09-13T14:29:29.956290+0000","last_fullsized":"2024-09-13T14:29:29.956290+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.12","version":"162'13556","reported_seq":18209,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:47.181823+0000","last_change":"2024-09-13T14:28:59.430905+0000","last_active":"2024-09-13T14:29:47.181823+0000","last_peered":"2024-09-13T14:29:47.181823+0000","last_clean":"2024-09-13T14:29:47.181823+0000","last_became_active":"2024-09-13T14:28:59.398490+0000","last_became_peered":"2024-09-13T14:28:59.398490+0000","last_unstale":"2024-09-13T14:29:47.181823+0000","last_undegraded":"2024-09-13T14:29:47.181823+0000","last_fullsized":"2024-09-13T14:29:47.181823+0000","mapping_epoch":139,"log_start":"109'2909","ondisk_log_start":"109'2909","created":119,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":5,"last_scrub":"123'10960","last_scrub_stamp":"2024-09-13T14:28:01.403797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:28:01.403797+0000","objects_scrubbed":2775,"log_size":10647,"log_dups_size":0,"ondisk_log_size":10647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:03.100834+0000","scrub_duration":0.60360896500000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11337728,"num_objects":173,"num_object_clones":0,"num_object_copies":519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2602,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1467,"num_bytes_recovered":96141312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956363+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:29:29.956363+0000","last_peered":"2024-09-13T14:29:29.956363+0000","last_clean":"2024-09-13T14:29:29.956363+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:29:29.956363+0000","last_undegraded":"2024-09-13T14:29:29.956363+0000","last_fullsized":"2024-09-13T14:29:29.956363+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":132,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956557+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:29:29.956557+0000","last_peered":"2024-09-13T14:29:29.956557+0000","last_clean":"2024-09-13T14:29:29.956557+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:29:29.956557+0000","last_undegraded":"2024-09-13T14:29:29.956557+0000","last_fullsized":"2024-09-13T14:29:29.956557+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.13","version":"162'13598","reported_seq":17492,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:43.833798+0000","last_change":"2024-09-13T14:29:04.416772+0000","last_active":"2024-09-13T14:29:43.833798+0000","last_peered":"2024-09-13T14:29:43.833798+0000","last_clean":"2024-09-13T14:29:43.833798+0000","last_became_active":"2024-09-13T14:29:04.391523+0000","last_became_peered":"2024-09-13T14:29:04.391523+0000","last_unstale":"2024-09-13T14:29:43.833798+0000","last_undegraded":"2024-09-13T14:29:43.833798+0000","last_fullsized":"2024-09-13T14:29:43.833798+0000","mapping_epoch":144,"log_start":"109'2099","ondisk_log_start":"109'2099","created":119,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":5,"last_scrub":"121'10896","last_scrub_stamp":"2024-09-13T14:27:58.015457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:58.015457+0000","objects_scrubbed":2982,"log_size":11499,"log_dups_size":0,"ondisk_log_size":11499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:17:30.782828+0000","scrub_duration":1153,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16908288,"num_objects":258,"num_object_clones":0,"num_object_copies":774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1232,"num_bytes_recovered":80740352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978193+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:29:38.978193+0000","last_peered":"2024-09-13T14:29:38.978193+0000","last_clean":"2024-09-13T14:29:38.978193+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:29:38.978193+0000","last_undegraded":"2024-09-13T14:29:38.978193+0000","last_fullsized":"2024-09-13T14:29:38.978193+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":129,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927461+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:29:29.927461+0000","last_peered":"2024-09-13T14:29:29.927461+0000","last_clean":"2024-09-13T14:29:29.927461+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:29:29.927461+0000","last_undegraded":"2024-09-13T14:29:29.927461+0000","last_fullsized":"2024-09-13T14:29:29.927461+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.10","version":"162'13575","reported_seq":21878,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.890711+0000","last_change":"2024-09-13T14:28:14.385882+0000","last_active":"2024-09-13T14:29:42.890711+0000","last_peered":"2024-09-13T14:29:42.890711+0000","last_clean":"2024-09-13T14:29:42.890711+0000","last_became_active":"2024-09-13T14:28:14.371527+0000","last_became_peered":"2024-09-13T14:28:14.371527+0000","last_unstale":"2024-09-13T14:29:42.890711+0000","last_undegraded":"2024-09-13T14:29:42.890711+0000","last_fullsized":"2024-09-13T14:29:42.890711+0000","mapping_epoch":130,"log_start":"109'1900","ondisk_log_start":"109'1900","created":119,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'10878","last_scrub_stamp":"2024-09-13T14:27:56.114518+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.114518+0000","objects_scrubbed":2991,"log_size":11675,"log_dups_size":0,"ondisk_log_size":11675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:11.102876+0000","scrub_duration":984,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18743296,"num_objects":286,"num_object_clones":0,"num_object_copies":858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2705,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2715,"num_bytes_recovered":177930240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927669+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:29:29.927669+0000","last_peered":"2024-09-13T14:29:29.927669+0000","last_clean":"2024-09-13T14:29:29.927669+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:29:29.927669+0000","last_undegraded":"2024-09-13T14:29:29.927669+0000","last_fullsized":"2024-09-13T14:29:29.927669+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965883+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:29:38.965883+0000","last_peered":"2024-09-13T14:29:38.965883+0000","last_clean":"2024-09-13T14:29:38.965883+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:29:38.965883+0000","last_undegraded":"2024-09-13T14:29:38.965883+0000","last_fullsized":"2024-09-13T14:29:38.965883+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.11","version":"162'13397","reported_seq":18665,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.831802+0000","last_change":"2024-09-13T14:28:24.423400+0000","last_active":"2024-09-13T14:29:42.831802+0000","last_peered":"2024-09-13T14:29:42.831802+0000","last_clean":"2024-09-13T14:29:42.831802+0000","last_became_active":"2024-09-13T14:28:24.404903+0000","last_became_peered":"2024-09-13T14:28:24.404903+0000","last_unstale":"2024-09-13T14:29:42.831802+0000","last_undegraded":"2024-09-13T14:29:42.831802+0000","last_fullsized":"2024-09-13T14:29:42.831802+0000","mapping_epoch":132,"log_start":"109'3300","ondisk_log_start":"109'3300","created":119,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":5,"last_scrub":"121'10852","last_scrub_stamp":"2024-09-13T14:27:58.806730+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:58.806730+0000","objects_scrubbed":2866,"log_size":10097,"log_dups_size":0,"ondisk_log_size":10097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:01.388283+0000","scrub_duration":0.93837820299999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20054016,"num_objects":306,"num_object_clones":0,"num_object_copies":918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2560,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2521,"num_bytes_recovered":165216256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":186,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972433+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:29:38.972433+0000","last_peered":"2024-09-13T14:29:38.972433+0000","last_clean":"2024-09-13T14:29:38.972433+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:29:38.972433+0000","last_undegraded":"2024-09-13T14:29:38.972433+0000","last_fullsized":"2024-09-13T14:29:38.972433+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965158+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:29:38.965158+0000","last_peered":"2024-09-13T14:29:38.965158+0000","last_clean":"2024-09-13T14:29:38.965158+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:29:38.965158+0000","last_undegraded":"2024-09-13T14:29:38.965158+0000","last_fullsized":"2024-09-13T14:29:38.965158+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.924560+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:29:29.924560+0000","last_peered":"2024-09-13T14:29:29.924560+0000","last_clean":"2024-09-13T14:29:29.924560+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:29:29.924560+0000","last_undegraded":"2024-09-13T14:29:29.924560+0000","last_fullsized":"2024-09-13T14:29:29.924560+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":208,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965328+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:29:38.965328+0000","last_peered":"2024-09-13T14:29:38.965328+0000","last_clean":"2024-09-13T14:29:38.965328+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:29:38.965328+0000","last_undegraded":"2024-09-13T14:29:38.965328+0000","last_fullsized":"2024-09-13T14:29:38.965328+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":188,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978207+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:29:38.978207+0000","last_peered":"2024-09-13T14:29:38.978207+0000","last_clean":"2024-09-13T14:29:38.978207+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:29:38.978207+0000","last_undegraded":"2024-09-13T14:29:38.978207+0000","last_fullsized":"2024-09-13T14:29:38.978207+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":177,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.978602+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:29:38.978602+0000","last_peered":"2024-09-13T14:29:38.978602+0000","last_clean":"2024-09-13T14:29:38.978602+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:29:38.978602+0000","last_undegraded":"2024-09-13T14:29:38.978602+0000","last_fullsized":"2024-09-13T14:29:38.978602+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965143+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:29:38.965143+0000","last_peered":"2024-09-13T14:29:38.965143+0000","last_clean":"2024-09-13T14:29:38.965143+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:29:38.965143+0000","last_undegraded":"2024-09-13T14:29:38.965143+0000","last_fullsized":"2024-09-13T14:29:38.965143+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":145,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956462+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:29:29.956462+0000","last_peered":"2024-09-13T14:29:29.956462+0000","last_clean":"2024-09-13T14:29:29.956462+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:29:29.956462+0000","last_undegraded":"2024-09-13T14:29:29.956462+0000","last_fullsized":"2024-09-13T14:29:29.956462+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":210,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.969002+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:29:38.969002+0000","last_peered":"2024-09-13T14:29:38.969002+0000","last_clean":"2024-09-13T14:29:38.969002+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:29:38.969002+0000","last_undegraded":"2024-09-13T14:29:38.969002+0000","last_fullsized":"2024-09-13T14:29:38.969002+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":179,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972154+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:29:38.972154+0000","last_peered":"2024-09-13T14:29:38.972154+0000","last_clean":"2024-09-13T14:29:38.972154+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:29:38.972154+0000","last_undegraded":"2024-09-13T14:29:38.972154+0000","last_fullsized":"2024-09-13T14:29:38.972154+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":143,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956296+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:29:29.956296+0000","last_peered":"2024-09-13T14:29:29.956296+0000","last_clean":"2024-09-13T14:29:29.956296+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:29:29.956296+0000","last_undegraded":"2024-09-13T14:29:29.956296+0000","last_fullsized":"2024-09-13T14:29:29.956296+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":128,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.926994+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:29:29.926994+0000","last_peered":"2024-09-13T14:29:29.926994+0000","last_clean":"2024-09-13T14:29:29.926994+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:29:29.926994+0000","last_undegraded":"2024-09-13T14:29:29.926994+0000","last_fullsized":"2024-09-13T14:29:29.926994+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":143,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.956273+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:29:29.956273+0000","last_peered":"2024-09-13T14:29:29.956273+0000","last_clean":"2024-09-13T14:29:29.956273+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:29:29.956273+0000","last_undegraded":"2024-09-13T14:29:29.956273+0000","last_fullsized":"2024-09-13T14:29:29.956273+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":111,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.927342+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:29:29.927342+0000","last_peered":"2024-09-13T14:29:29.927342+0000","last_clean":"2024-09-13T14:29:29.927342+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:29:29.927342+0000","last_undegraded":"2024-09-13T14:29:29.927342+0000","last_fullsized":"2024-09-13T14:29:29.927342+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.18","version":"162'13413","reported_seq":16182,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:42.969779+0000","last_change":"2024-09-13T14:29:23.595588+0000","last_active":"2024-09-13T14:29:42.969779+0000","last_peered":"2024-09-13T14:29:42.969779+0000","last_clean":"2024-09-13T14:29:42.969779+0000","last_became_active":"2024-09-13T14:29:23.568013+0000","last_became_peered":"2024-09-13T14:29:23.568013+0000","last_unstale":"2024-09-13T14:29:42.969779+0000","last_undegraded":"2024-09-13T14:29:42.969779+0000","last_fullsized":"2024-09-13T14:29:42.969779+0000","mapping_epoch":157,"log_start":"109'3062","ondisk_log_start":"109'3062","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'10803","last_scrub_stamp":"2024-09-13T14:27:56.776539+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:25:28.084709+0000","last_clean_scrub_stamp":"2024-09-13T14:27:56.776539+0000","objects_scrubbed":2943,"log_size":10351,"log_dups_size":0,"ondisk_log_size":10351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:54.552990+0000","scrub_duration":992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2630,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":858,"num_bytes_recovered":56229888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":211,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.966133+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:29:38.966133+0000","last_peered":"2024-09-13T14:29:38.966133+0000","last_clean":"2024-09-13T14:29:38.966133+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:29:38.966133+0000","last_undegraded":"2024-09-13T14:29:38.966133+0000","last_fullsized":"2024-09-13T14:29:38.966133+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":196,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.965592+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:29:38.965592+0000","last_peered":"2024-09-13T14:29:38.965592+0000","last_clean":"2024-09-13T14:29:38.965592+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:29:38.965592+0000","last_undegraded":"2024-09-13T14:29:38.965592+0000","last_fullsized":"2024-09-13T14:29:38.965592+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"162'16439","reported_seq":19122,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:47.173954+0000","last_change":"2024-09-13T14:29:29.954153+0000","last_active":"2024-09-13T14:29:47.173954+0000","last_peered":"2024-09-13T14:29:47.173954+0000","last_clean":"2024-09-13T14:29:47.173954+0000","last_became_active":"2024-09-13T14:29:29.929582+0000","last_became_peered":"2024-09-13T14:29:29.929582+0000","last_unstale":"2024-09-13T14:29:47.173954+0000","last_undegraded":"2024-09-13T14:29:47.173954+0000","last_fullsized":"2024-09-13T14:29:47.173954+0000","mapping_epoch":160,"log_start":"114'8817","ondisk_log_start":"114'8817","created":119,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":5,"last_scrub":"129'13917","last_scrub_stamp":"2024-09-13T14:28:13.099709+0000","last_deep_scrub":"129'13917","last_deep_scrub_stamp":"2024-09-13T14:28:13.099709+0000","last_clean_scrub_stamp":"2024-09-13T14:28:13.099709+0000","objects_scrubbed":2812,"log_size":7622,"log_dups_size":0,"ondisk_log_size":7622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":13,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:29.950232+0000","scrub_duration":12.254733592999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11075584,"num_objects":169,"num_object_clones":0,"num_object_copies":507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2643,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":821,"num_bytes_recovered":53805056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":161,"state":"active+clean","last_fresh":"2024-09-13T14:29:29.955961+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:29:29.955961+0000","last_peered":"2024-09-13T14:29:29.955961+0000","last_clean":"2024-09-13T14:29:29.955961+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:29:29.955961+0000","last_undegraded":"2024-09-13T14:29:29.955961+0000","last_fullsized":"2024-09-13T14:29:29.955961+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":190,"reported_epoch":162,"state":"active+clean","last_fresh":"2024-09-13T14:29:38.972253+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:29:38.972253+0000","last_peered":"2024-09-13T14:29:38.972253+0000","last_clean":"2024-09-13T14:29:38.972253+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:29:38.972253+0000","last_undegraded":"2024-09-13T14:29:38.972253+0000","last_fullsized":"2024-09-13T14:29:38.972253+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":508035096,"num_objects":7753,"num_object_clones":0,"num_object_copies":23259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7753,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152009,"num_write_kb":3225345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20480,"num_bytes_recovered":1342111744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1820405760,"data_stored":1820393544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":279208,"ondisk_log_size":279208,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397723,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534952,"kb_used_data":233440,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93210200,"statfs":{"total":95995035648,"available":95447244800,"internally_reserved":0,"allocated":239042560,"data_stored":238144913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593951,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520724,"kb_used_data":219212,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93224428,"statfs":{"total":95995035648,"available":95461814272,"internally_reserved":0,"allocated":224473088,"data_stored":223595203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790181,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":771724,"kb_used_data":457732,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92973428,"statfs":{"total":95995035648,"available":95204790272,"internally_reserved":0,"allocated":468717568,"data_stored":467843118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986409,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":609308,"kb_used_data":311956,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93135844,"statfs":{"total":95995035648,"available":95371104256,"internally_reserved":0,"allocated":319442944,"data_stored":318556947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696967,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343496,"kb_used_data":199424,"kb_used_omap":4,"kb_used_meta":144059,"kb_avail":93401656,"statfs":{"total":95995035648,"available":95643295744,"internally_reserved":0,"allocated":204210176,"data_stored":203339871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":147516491},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860494,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278416,"kb_used_data":134088,"kb_used_omap":8,"kb_used_meta":144119,"kb_avail":93466736,"statfs":{"total":95995035648,"available":95709937664,"internally_reserved":0,"allocated":137306112,"data_stored":136428161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":147578700},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024021,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":361032,"kb_used_data":176640,"kb_used_omap":7,"kb_used_meta":184376,"kb_avail":93384120,"statfs":{"total":95995035648,"available":95625338880,"internally_reserved":0,"allocated":180879360,"data_stored":180008976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":188801463},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187556,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":327500,"kb_used_data":143044,"kb_used_omap":7,"kb_used_meta":184376,"kb_avail":93417652,"statfs":{"total":95995035648,"available":95659675648,"internally_reserved":0,"allocated":146477056,"data_stored":145603394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":188801355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":134610944,"data_stored":134610944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":166789120,"data_stored":166789120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":131928064,"data_stored":131923992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":195362816,"data_stored":195362816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":317194240,"data_stored":317194240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":445583360,"data_stored":445579288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":202702848,"data_stored":202702848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":226234368,"data_stored":226230296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:29:50.116 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:29:50.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-13T14:29:50.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:50.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:50.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.296 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:50.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:50.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:50.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:29:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:29:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:29:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:50 smithi189 ceph-mon[90992]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:50.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:50.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:50.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:50 smithi017 ceph-mon[103001]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:50 smithi017 ceph-mon[107556]: from='client.35265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:29:50.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:50.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:50.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:50.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:50.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:29:50.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:51.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:51.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:51.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:51.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:51.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:51.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:51.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:51.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:51.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:51.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:51.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:51.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:51 smithi017 ceph-mon[103001]: pgmap v432: 99 pgs: 99 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2024-09-13T14:29:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:51 smithi017 ceph-mon[107556]: pgmap v432: 99 pgs: 99 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2024-09-13T14:29:51.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:51.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:51 smithi189 ceph-mon[90992]: pgmap v432: 99 pgs: 99 active+clean; 318 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2024-09-13T14:29:51.834 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:51.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:51.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:51.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:52.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:52.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:52.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:52.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:52.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:52.416 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:52.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:52.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:52.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:52.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:52.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:52.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:52.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:52.834 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 134626 objects 2024-09-13T14:29:52.834 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :161.932 2024-09-13T14:29:52.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:52.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:52.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:53.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:53.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:53.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:53.198 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:53.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:53.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:53.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:53.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:53.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:53.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:53.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:53.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:53.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:53.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:53 smithi017 ceph-mon[103001]: pgmap v433: 99 pgs: 99 active+clean; 243 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:29:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:53 smithi017 ceph-mon[107556]: pgmap v433: 99 pgs: 99 active+clean; 243 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:29:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:53 smithi189 ceph-mon[90992]: pgmap v433: 99 pgs: 99 active+clean; 243 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:29:53.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:53.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:53.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:53.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:53.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:54.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:54.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:54.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:54.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:54.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:54.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:54.416 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:54.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:54.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:54.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:54.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:54.641 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:54.646 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:54.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:54.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:54.786 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:54.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:54.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:54.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:54.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:54.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:55.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:55.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:55.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:55.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:55.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:55.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:55.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:55.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:55.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:55.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:55.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:29:55.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:55 smithi017 ceph-mon[103001]: pgmap v434: 99 pgs: 99 active+clean; 84 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:29:55.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:55 smithi017 ceph-mon[107556]: pgmap v434: 99 pgs: 99 active+clean; 84 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:29:55.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:55.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:55 smithi189 ceph-mon[90992]: pgmap v434: 99 pgs: 99 active+clean; 84 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:29:55.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:55.800 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:55.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:55.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:55.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:56.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:56.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:56.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:56.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:56.173 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:56.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:56.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:56.390 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:29:56.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:56.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:56.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:56.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:56.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:56.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:56.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:56.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1325623947' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:56 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1325623947' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:56 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1325623947' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:56 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:29:56.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:56.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:56.899 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:56.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:56.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:57.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:29:57.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:57.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:29:57.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:57.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:57.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:57.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:57.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:57.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:57.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:57.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:57.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:57.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:57.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:57.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:29:57] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: pgmap v435: 99 pgs: 99 active+clean; 41 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: pgmap v435: 99 pgs: 99 active+clean; 41 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: pgmap v435: 99 pgs: 99 active+clean; 41 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:29:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:29:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:29:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:29:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:29:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:29:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:57.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:57.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:57.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:57.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:57.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:57.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:58.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:58.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:58.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:58.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:58.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:29:58.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.314 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:58.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:29:58.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:58.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:58.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:58.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:58.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:58.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:58.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:58.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:58.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:58.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:58.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:58.951 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:29:58.951 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:29:58.951 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:29:58.951 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:29:58.951 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:29:59.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:59.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:59.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:59.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:29:59.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:29:59.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:29:59.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:29:59.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:29:59.498 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:29:59 smithi017 ceph-mon[103001]: pgmap v437: 99 pgs: 99 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-13T14:29:59.498 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:29:59 smithi017 ceph-mon[107556]: pgmap v437: 99 pgs: 99 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-13T14:29:59.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:29:59.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:29:59.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:29:59.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:29:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:29:59 smithi189 ceph-mon[90992]: pgmap v437: 99 pgs: 99 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-13T14:29:59.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:29:59.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:29:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:29:59.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:29:59.924 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:29:59.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:29:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:29:59.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:00.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:00.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:00.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:00.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:00.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:00.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:00.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:00.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:30:00.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:00.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:00.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:00 smithi189 ceph-mon[90992]: overall HEALTH_OK 2024-09-13T14:30:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:00.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:00.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:00.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:00 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T14:30:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:00 smithi017 ceph-mon[107556]: overall HEALTH_OK 2024-09-13T14:30:00.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:00.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:00.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:00.877 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:00.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:00.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:00.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:01.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:01.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:01.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:01.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:01.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:01.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:01.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:01.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:01.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:01.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:01.634 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:01.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:01 smithi017 ceph-mon[103001]: pgmap v438: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-09-13T14:30:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:01 smithi017 ceph-mon[107556]: pgmap v438: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-09-13T14:30:01.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:01.747 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:01 smithi189 ceph-mon[90992]: pgmap v438: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-09-13T14:30:01.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:01.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:01.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:01.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:01.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:01.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:02.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:02.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:02.186 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.209 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:02.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:02.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:02.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:02.399 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 11 2024-09-13T14:30:02.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:02.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:02.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:02.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:02.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2183126490' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2183126490' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:02.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2183126490' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:30:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:02.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:02.813 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:02.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:02.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:02.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:02.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:02.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:03.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:03.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:03.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:03.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:03.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:03.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:03.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[103001]: pgmap v439: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 382 op/s 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[103001]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[107556]: pgmap v439: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 382 op/s 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:30:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:03 smithi017 ceph-mon[107556]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:30:03.457 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:03.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:03.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:03.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:03.692 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-13T14:30:03.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-13T14:30:03.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:03 smithi189 ceph-mon[90992]: pgmap v439: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 382 op/s 2024-09-13T14:30:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:03 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:30:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:03 smithi189 ceph-mon[90992]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:30:03.809 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:03.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:03.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:03.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:03.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:04.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:04.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:04.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:04.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:04.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:04.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:04.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:04.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:04.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:04.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:04.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:04.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:04.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:04.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:04.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:04.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:04.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:04.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:05.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:05.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:05.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:05.204 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:05.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:05.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:05.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:05.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:05.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.577 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:05.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:05.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:05.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[103001]: pgmap v441: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T14:30:05.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:30:05.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[103001]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:30:05.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[107556]: pgmap v441: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T14:30:05.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:30:05.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:05 smithi017 ceph-mon[107556]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:30:05.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:05 smithi189 ceph-mon[90992]: pgmap v441: 99 pgs: 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T14:30:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:05 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:30:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:05 smithi189 ceph-mon[90992]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:30:05.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:05.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:05.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:05.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:05.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:05.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:05.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:06.046 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:30:06.047 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:30:06.047 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:30:06.047 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:30:06.058 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:30:06.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:30:06.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:06.075 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:30:06.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:30:06.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:06.094 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:30:06.094 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:30:06.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:30:06.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:30:06.126 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:30:06.126 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:30:06.142 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:30:06.142 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:30:06.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:06.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:06.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:06.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:06.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:06.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:06.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:06.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:06.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:06 smithi017 ceph-mon[103001]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:30:06.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/507740206' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:06.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:06 smithi017 ceph-mon[107556]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:30:06.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/507740206' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:06.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:06.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:06.710 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:06.710 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:06 smithi189 ceph-mon[90992]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:30:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/507740206' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:06.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:06.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:06.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:06.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:06.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:07.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:07.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:07.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:07.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:07.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:07.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:07.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:07.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:07.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:07.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:07.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:07.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[103001]: pgmap v444: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[103001]: 1.0 scrub starts 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[103001]: 1.0 scrub ok 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[107556]: pgmap v444: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[107556]: 1.0 scrub starts 2024-09-13T14:30:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-mon[107556]: 1.0 scrub ok 2024-09-13T14:30:07.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:07] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:30:07.715 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:07.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:07 smithi189 ceph-mon[90992]: pgmap v444: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:07 smithi189 ceph-mon[90992]: 1.0 scrub starts 2024-09-13T14:30:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:07 smithi189 ceph-mon[90992]: 1.0 scrub ok 2024-09-13T14:30:07.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:07.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:07.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:07.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:07.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:07.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:08.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:08.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:08.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:08.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:08.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:08.345 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:08.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:30:08.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:08.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:08.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:08.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:08.693 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:30:08.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:30:08.693 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T14:30:08.693 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2024-09-13T14:30:08.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-13T14:30:08.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:08.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:08.857 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:08.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:08.870 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:08.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:08.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:08.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:09.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:09.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:09.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:09.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:09.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:09.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:09.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:09.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:09.417 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:30:09.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:30:09.494 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:09.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:09.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:09 smithi017 ceph-mon[103001]: pgmap v445: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:09.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:30:09.533 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:09 smithi017 ceph-mon[107556]: pgmap v445: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:09.533 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:30:09.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:09.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:09.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:09 smithi189 ceph-mon[90992]: pgmap v445: 109 pgs: 10 unknown, 99 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:30:09.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:09.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:09.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:09.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:09.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:10.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:10.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:10.145 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:10.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:10.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:10.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:10.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:10.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:10.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:10 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:30:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:10 smithi189 ceph-mon[90992]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:30:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:10.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:10.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:10.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:30:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[103001]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:30:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2205332809' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:30:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[107556]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:30:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:30:10.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:10.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:10.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:10.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:10.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:10.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:11.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:11.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:11.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:11.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:11.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:11.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:11.402 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:11.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:11 smithi189 ceph-mon[90992]: pgmap v447: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:30:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:11 smithi189 ceph-mon[90992]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:30:11.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:11.563 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:11.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:11.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[103001]: pgmap v447: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:30:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[103001]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:30:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[107556]: pgmap v447: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:30:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:11 smithi017 ceph-mon[107556]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:30:11.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:11.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:11.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:11.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:11.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:12.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:12.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:12.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:12.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:12.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:12.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:12.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:12.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:12.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:12.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:12.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:12.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:12.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:12.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[107556]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[103001]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:12.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:12.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:12 smithi189 ceph-mon[90992]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:30:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:12 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:12.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:12.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:12.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:12.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:12.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:13.021 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:13.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:13.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:13.162 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:13.162 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:13.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:13.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:13.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:13.397 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:13.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:13.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:13.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:13.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:13 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:13 smithi017 ceph-mon[103001]: pgmap v450: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:13 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:13 smithi017 ceph-mon[107556]: pgmap v450: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:13.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:13 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:13 smithi189 ceph-mon[90992]: pgmap v450: 109 pgs: 109 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:13.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:13.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:13.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:30:13.903 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:13.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:13.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:30:14.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:14.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:14.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:14.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:14.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:14.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:14.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:14.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:14.459 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:14.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:14.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:14.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:14.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:14.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:14.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:14.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:14.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:14.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:14.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:15.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:15.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:15.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:15.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:15.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:15.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:15.406 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:15.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:15.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:15.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:15.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:15.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:15 smithi017 ceph-mon[103001]: pgmap v451: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2024-09-13T14:30:15.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:15 smithi017 ceph-mon[107556]: pgmap v451: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2024-09-13T14:30:15.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:15 smithi189 ceph-mon[90992]: pgmap v451: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2024-09-13T14:30:15.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:15.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:15.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:15.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:15.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:15.929 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:16.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:16.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:16.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:16.140 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:30:16.140 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:30:16.157 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:30:16.157 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:30:16.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:16.174 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:30:16.174 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:30:16.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:30:16.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:30:16.206 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:30:16.206 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:30:16.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:30:16.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:30:16.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:30:16.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:30:16.254 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:30:16.254 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:30:16.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:16.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:16.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:16.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:16.455 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' removed 2024-09-13T14:30:16.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:16.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:16.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:16.643 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:16.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4256856376' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2852942144' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4256856376' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2852942144' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4256856376' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:16 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:16.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2852942144' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:16.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:16.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:16.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:16.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:16.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:17.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:17.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:17.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:17.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:17.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:17.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:17.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:17.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:17.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:17.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:17.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:17.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:17.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:17.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:17] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[103001]: pgmap v452: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[103001]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[107556]: pgmap v452: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:30:17.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:17.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[107556]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:30:17.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:17.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:17 smithi189 ceph-mon[90992]: pgmap v452: 109 pgs: 109 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-09-13T14:30:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:17 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:17 smithi189 ceph-mon[90992]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:30:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:17 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:17.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:17.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:17.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:17.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:17.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:18.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:18.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:18.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:18.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:18.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:18.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:18.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:18.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:18.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:18.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:18.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:18.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:18.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:18.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:18.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:18.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:18.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:19.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:19.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:19.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:19.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:19.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:19.172 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:30:19.173 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:30:19.174 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-13T14:30:19.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-13T14:30:19.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:19.284 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:19.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:19.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:19.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:19.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:19.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:19.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:19.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:19.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:19 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:19.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:19 smithi017 ceph-mon[107556]: pgmap v454: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:19.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:19 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:19.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:19 smithi017 ceph-mon[103001]: pgmap v454: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:19.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:19 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:19.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:19 smithi189 ceph-mon[90992]: pgmap v454: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:19.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:19.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:19.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:19.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:19.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:20.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:20.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:20.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:20.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:20.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:20.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:20.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:20.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:20.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:20.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:20.466 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-13T14:30:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/205055188' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:20 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:20.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:20.569 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/205055188' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.569 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:20 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/205055188' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:20 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:30:20.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:20.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:20.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:20.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:20.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:20.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:21.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:21.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:21.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:21.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:21.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:21.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:21.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:21.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:21.470 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:30:21.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:21.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:21.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:21.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[103001]: pgmap v455: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:21.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[103001]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[107556]: pgmap v455: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[107556]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:30:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:21.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:21 smithi189 ceph-mon[90992]: pgmap v455: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:30:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:21 smithi189 ceph-mon[90992]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:30:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:21.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:21.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:21.855 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:21.896 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:21.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:21.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:22.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:22.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:22.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:30:22.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:22.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:22.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:22.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:22.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:22.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:22.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:22.530 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:22.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:22.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:22.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[107556]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:22 smithi017 ceph-mon[103001]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:30:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:22 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/937604166' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:22.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:22 smithi189 ceph-mon[90992]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:30:22.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:22.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:22.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:22.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:22.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:23.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:23.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:23.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:23.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:23.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:23.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:23.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:23.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:23.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:23.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:23.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:23.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:23.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:23 smithi017 ceph-mon[103001]: pgmap v458: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:23 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:23 smithi017 ceph-mon[107556]: pgmap v458: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:23 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:23 smithi189 ceph-mon[90992]: pgmap v458: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 4.5 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:23 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:23.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:23.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:23.831 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:30:23.832 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:30:23.832 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:30:23.832 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:30:23.832 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:30:23.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:23.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:23.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:24.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:24.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:24.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:24.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:24.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:24.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:24.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:24.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:24.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:24.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:24.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:30:24.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:24.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:24.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:24.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:24.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:24.733 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:24 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:24.733 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:24 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:24 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:24.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:24.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:24.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:24.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:24.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:25.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:25.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:25.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:25.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:25.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:25.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:25.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:25.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:25.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:25 smithi017 ceph-mon[103001]: pgmap v459: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:25 smithi017 ceph-mon[107556]: pgmap v459: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:25.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:25.728 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:25.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:25 smithi189 ceph-mon[90992]: pgmap v459: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:25.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:25.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:25.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:25.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:25.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:25.995 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:26.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:26.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:26.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:26.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:26.285 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:30:26.285 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:30:26.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:26.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:26.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:26.490 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' created 2024-09-13T14:30:26.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/922252429' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/922252429' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:26.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:26.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:26.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/922252429' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:30:26.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:26.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:26.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:26.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:30:26.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:27.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:27.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:27.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:27.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:27.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:27.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:27.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:27.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:27.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:27.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:27.641 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:27.658 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:27] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-13T14:30:27.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[103001]: pgmap v460: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:27.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:30:27.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[103001]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:30:27.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:27.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[107556]: pgmap v460: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:27.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:30:27.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[107556]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:30:27.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:27.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:27 smithi189 ceph-mon[90992]: pgmap v460: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:27 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:30:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:27 smithi189 ceph-mon[90992]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:30:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:27.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:27.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:27.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:27.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:27.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:28.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:28.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:28.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-13T14:30:28.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:28.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:28.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:28.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:28.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:28.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:28.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:28.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:28.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:28.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:28.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:28 smithi189 ceph-mon[90992]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:30:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:28 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:28.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:28.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:28.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:28.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:28 smithi017 ceph-mon[107556]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:30:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:28 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:28 smithi017 ceph-mon[103001]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:30:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:28 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:28.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:29.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:29.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:29.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:29.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:29.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:29.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:29.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:29.494 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:29.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:29.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[103001]: pgmap v463: 99 pgs: 12 unknown, 4 creating+peering, 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:29.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:29.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[103001]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:30:29.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[107556]: pgmap v463: 99 pgs: 12 unknown, 4 creating+peering, 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:29.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:29.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:29 smithi017 ceph-mon[107556]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:30:29.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:29.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:29.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:29 smithi189 ceph-mon[90992]: pgmap v463: 99 pgs: 12 unknown, 4 creating+peering, 1 active+clean+scrubbing, 82 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:29 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:29 smithi189 ceph-mon[90992]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:30:29.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:29.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:29.815 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:30:29.815 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:30:29.820 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:29.835 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:30:29.835 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:30:29.853 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:30:29.853 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:30:29.869 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:30:29.870 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:30:29.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:29.885 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:30:29.886 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:30:29.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:29.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:29.902 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:30:29.902 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:30:29.918 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:30:29.918 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:30:29.933 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:30:29.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:30:30.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:30.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:30.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:30.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:30.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:30.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:30.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:30.282 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:30.282 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:30.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:30.398 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:30.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:30.503 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:30.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:30.562 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:30.562 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2514076951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:30.562 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[103001]: pgmap v465: 99 pgs: 1 unknown, 1 active+clean+scrubbing, 4 creating+peering, 93 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.562 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:30.563 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2514076951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:30.563 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:30 smithi017 ceph-mon[107556]: pgmap v465: 99 pgs: 1 unknown, 1 active+clean+scrubbing, 4 creating+peering, 93 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:30.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:30.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:30.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:30 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2514076951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:30 smithi189 ceph-mon[90992]: pgmap v465: 99 pgs: 1 unknown, 1 active+clean+scrubbing, 4 creating+peering, 93 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:30.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:30.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:30.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:30.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:30.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:30.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:30:30.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:31.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:31.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:31.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:31.246 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:31.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:31.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:31.342 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:31.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:31.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:31.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:31.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:31.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:31.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:31.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:31.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:31.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:31.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:31.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:32.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:32.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:32.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:32.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:32.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:32.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:32.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:32.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:32.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:32.526 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:32.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:32.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:32.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:32.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:32.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:32.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:32.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:32.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:32.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:32 smithi189 ceph-mon[90992]: pgmap v466: 99 pgs: 1 active+clean+scrubbing, 4 creating+peering, 94 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:32.936 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:32.936 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:33.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:33.042 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:33.144 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:33.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:32 smithi017 ceph-mon[107556]: pgmap v466: 99 pgs: 1 active+clean+scrubbing, 4 creating+peering, 94 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:32 smithi017 ceph-mon[103001]: pgmap v466: 99 pgs: 1 active+clean+scrubbing, 4 creating+peering, 94 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:33.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:33.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:33.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:33.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:33.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:33.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:30:33.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:33.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:33.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:33.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:30:33.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:33.831 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:33.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:33.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:33.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:34.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:34.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:34.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:34.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:34.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:34.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:34.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:34.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:34.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:34.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:34.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:34.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:34.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:34.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:34.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:34.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:34.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:34.889 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-13T14:30:34.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:34.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:34.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[103001]: pgmap v467: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:34.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:34.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:34.903 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[107556]: pgmap v467: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:34.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:34.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:34.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:34 smithi189 ceph-mon[90992]: pgmap v467: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:34.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:34 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:34.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:35.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:35.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:35.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:35.142 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:35.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:35.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:35.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:35.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:35.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:35.477 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:35.477 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:35.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.522 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:35.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:35.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:35.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:35.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:35.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:35.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:35.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:35.892 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:30:35.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:35.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:35.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:36.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:36.032 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:35 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:36.032 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:36.032 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:35 smithi189 ceph-mon[90992]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:30:36.032 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:30:36.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:30:36.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:36.159 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[103001]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/361499476' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[107556]: osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:30:36.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:30:36.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:36.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:36.342 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:36.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:36.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:36.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:36.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:36.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:36.618 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-13T14:30:36.619 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:30:36.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:30:36.621 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:30:36.621 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:30:36.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:36.671 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:30:36.671 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:30:36.671 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_185083 2024-09-13T14:30:36.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:30:36.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:30:36.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:36.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:36.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:36.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:36.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:36.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:36.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:37.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:37.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[103001]: pgmap v469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:30:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[103001]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:30:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[107556]: pgmap v469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:30:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:36 smithi017 ceph-mon[107556]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:30:37.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:36 smithi189 ceph-mon[90992]: pgmap v469: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:36 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:36 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:30:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2550632898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:30:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:36 smithi189 ceph-mon[90992]: osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:30:37.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:37.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:37.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:37.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:37.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:37.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:37.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:37.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3711 3695 230.924 230.938 0.0033565 0.00431466 2024-09-13T14:30:37.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:37] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-13T14:30:37.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:37.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:37.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:37.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:37.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:37.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:38.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:38.142 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:30:38.142 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.5 scrub ok 2024-09-13T14:30:38.142 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.2 scrub starts 2024-09-13T14:30:38.142 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.2 scrub ok 2024-09-13T14:30:38.143 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.7 scrub starts 2024-09-13T14:30:38.143 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:37 smithi189 ceph-mon[90992]: 1.7 scrub ok 2024-09-13T14:30:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:30:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.5 scrub ok 2024-09-13T14:30:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.2 scrub starts 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.2 scrub ok 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.7 scrub starts 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[103001]: 1.7 scrub ok 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.5 scrub ok 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.2 scrub starts 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.2 scrub ok 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.7 scrub starts 2024-09-13T14:30:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:37 smithi017 ceph-mon[107556]: 1.7 scrub ok 2024-09-13T14:30:38.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:38.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:38.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:38.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:38.424 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:38.424 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:38.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:38.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:38.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:38.671 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6919 6903 215.701 200.5 0.00424776 0.00462636 2024-09-13T14:30:38.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:38.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:38.813 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:38.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:38.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:38.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:39.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:39.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:38 smithi189 ceph-mon[90992]: pgmap v471: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 43 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s wr, 76 op/s 2024-09-13T14:30:39.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:38 smithi189 ceph-mon[90992]: 1.6 scrub starts 2024-09-13T14:30:39.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:38 smithi189 ceph-mon[90992]: 1.6 scrub ok 2024-09-13T14:30:39.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3231618655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:39.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:39.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[103001]: pgmap v471: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 43 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s wr, 76 op/s 2024-09-13T14:30:39.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[103001]: 1.6 scrub starts 2024-09-13T14:30:39.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[103001]: 1.6 scrub ok 2024-09-13T14:30:39.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3231618655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:39.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[107556]: pgmap v471: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 43 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s wr, 76 op/s 2024-09-13T14:30:39.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[107556]: 1.6 scrub starts 2024-09-13T14:30:39.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[107556]: 1.6 scrub ok 2024-09-13T14:30:39.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3231618655' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:39.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:30:39.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:39.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:39.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:39.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:39.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:30:39.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:39.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:39.529 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:39.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:39.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:39.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9729 9713 202.336 175.625 0.00269474 0.00493406 2024-09-13T14:30:39.723 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:30:39.723 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:30:39.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:30:39.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:30:39.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:30:39.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:30:39.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:30:39.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:39.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:39.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:39.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:39.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:39.991 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:39 smithi189 ceph-mon[90992]: 1.a scrub starts 2024-09-13T14:30:39.991 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:39 smithi189 ceph-mon[90992]: 1.a scrub ok 2024-09-13T14:30:40.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:40.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:40.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:40.169 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:39 smithi017 ceph-mon[103001]: 1.a scrub starts 2024-09-13T14:30:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:39 smithi017 ceph-mon[103001]: 1.a scrub ok 2024-09-13T14:30:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:39 smithi017 ceph-mon[107556]: 1.a scrub starts 2024-09-13T14:30:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:39 smithi017 ceph-mon[107556]: 1.a scrub ok 2024-09-13T14:30:40.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:40.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:40.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:40.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:40.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:40.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:40.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:40.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:40.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:40.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:40.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12824 12808 200.106 193.438 0.00424686 0.00499075 2024-09-13T14:30:40.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:40.772 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:40.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:40.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:40.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:40.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:41.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.3 scrub ok 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: pgmap v472: 99 pgs: 99 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.4 scrub ok 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.1 scrub starts 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[103001]: 1.1 scrub ok 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:30:41.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.3 scrub ok 2024-09-13T14:30:41.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: pgmap v472: 99 pgs: 99 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s 2024-09-13T14:30:41.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:30:41.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.4 scrub ok 2024-09-13T14:30:41.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.1 scrub starts 2024-09-13T14:30:41.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:40 smithi017 ceph-mon[107556]: 1.1 scrub ok 2024-09-13T14:30:41.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:41.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:41.154 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.3 scrub ok 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: pgmap v472: 99 pgs: 99 active+clean; 277 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.4 scrub ok 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.1 scrub starts 2024-09-13T14:30:41.155 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:40 smithi189 ceph-mon[90992]: 1.1 scrub ok 2024-09-13T14:30:41.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:41.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:41.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:41.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:41.458 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:15019 6.d 6:be448ba7:::benchmark_data_smithi017_185083_object15018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:30:41.456062+0000", 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.00098143899999999997, 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0010129869999999999, 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:30:41.459 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 15019 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456062+0000", 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456062+0000", 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456061+0000", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456117+0000", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6079000000000003e-05 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456150+0000", 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2957999999999999e-05 2024-09-13T14:30:41.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456157+0000", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8970000000000002e-06 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456702+0000", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00054496299999999998 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456832+0000", 2024-09-13T14:30:41.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000129546 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.456914+0000", 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2570000000000004e-05 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:15007 6.a 6:571dbdd5:::benchmark_data_smithi017_185083_object15006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:30:41.451994+0000", 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0050500659999999998, 2024-09-13T14:30:41.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005155682, 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 15007 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.451994+0000", 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.451994+0000", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.451992+0000", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452024+0000", 2024-09-13T14:30:41.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1602e-05 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452066+0000", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1560000000000002e-05 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452072+0000", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4239999999999998e-06 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452144+0000", 2024-09-13T14:30:41.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2157999999999996e-05 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452473+0000", 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032872899999999999 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452596+0000", 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000122679 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:41.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.454353+0000", 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001757602 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.454684+0000", 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033043500000000002 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:15009 6.a 6:53f2d99f:::benchmark_data_smithi017_185083_object15008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:30:41.452443+0000", 2024-09-13T14:30:41.468 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0046008129999999996, 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048004090000000003, 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 15009 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452443+0000", 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452443+0000", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452442+0000", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452487+0000", 2024-09-13T14:30:41.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5293000000000002e-05 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452499+0000", 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1569999999999999e-05 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.452504+0000", 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6350000000000003e-06 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:41.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.453342+0000", 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00083780400000000004 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.453474+0000", 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013167100000000001 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.453702+0000", 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022795500000000001 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.454703+0000", 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0010013590000000001 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.454883+0000", 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018016700000000001 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:15018 6.d 6:b8a136b0:::benchmark_data_smithi017_185083_object15017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-09-13T14:30:41.473 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:30:41.455532+0000", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0015114200000000001, 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001803081, 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 15018 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455532+0000", 2024-09-13T14:30:41.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455532+0000", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455531+0000", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455618+0000", 2024-09-13T14:30:41.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7372000000000003e-05 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455633+0000", 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4731e-05 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455639+0000", 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.57e-06 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:41.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455689+0000", 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0012999999999997e-05 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455809+0000", 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000120018 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:41.455952+0000", 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014305399999999999 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:41.477 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.478 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:41.478 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:30:41.478 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 4 2024-09-13T14:30:41.478 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:30:41.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:41.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:41.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:41.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:41.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 15588 15572 194.632 172.75 0.00674217 0.00513167 2024-09-13T14:30:41.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:41.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:41.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:41.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:41.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:42.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:42.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:42.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:42.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:41 smithi017 ceph-mon[103001]: 1.9 scrub starts 2024-09-13T14:30:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:41 smithi017 ceph-mon[103001]: 1.9 scrub ok 2024-09-13T14:30:42.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:41 smithi017 ceph-mon[107556]: 1.9 scrub starts 2024-09-13T14:30:42.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:41 smithi017 ceph-mon[107556]: 1.9 scrub ok 2024-09-13T14:30:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:41 smithi189 ceph-mon[90992]: 1.9 scrub starts 2024-09-13T14:30:42.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:41 smithi189 ceph-mon[90992]: 1.9 scrub ok 2024-09-13T14:30:42.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:42.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:42.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:42.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:42.456 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:42.558 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:42.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:42.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 18057 18041 187.909 154.312 0.00976312 0.00531619 2024-09-13T14:30:42.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:42.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:42.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:30:42.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:42.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:42.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:42.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:42.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:42.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:42.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:42 smithi189 ceph-mon[90992]: pgmap v473: 99 pgs: 99 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s 2024-09-13T14:30:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:42 smithi189 ceph-mon[90992]: 1.8 scrub starts 2024-09-13T14:30:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:42 smithi189 ceph-mon[90992]: 1.8 scrub ok 2024-09-13T14:30:43.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[103001]: pgmap v473: 99 pgs: 99 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[103001]: 1.8 scrub starts 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[103001]: 1.8 scrub ok 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[107556]: pgmap v473: 99 pgs: 99 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 652 op/s 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:43.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[107556]: 1.8 scrub starts 2024-09-13T14:30:43.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:42 smithi017 ceph-mon[107556]: 1.8 scrub ok 2024-09-13T14:30:43.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:43.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:43.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:43.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:43.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:43.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:43.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:43.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:43.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:43.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:43.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 20184 20168 180.055 132.938 0.00151593 0.00554167 2024-09-13T14:30:43.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:43.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:43.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:43.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:43.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:43.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:43.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:44.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:44.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:30:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:44.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:44.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:44.275 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:30:44.277 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:30:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:44.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:44.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:44.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:44.502 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:44.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:44.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:44.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:44.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 22437 22421 175.141 140.812 0.00705927 0.00570333 2024-09-13T14:30:44.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:44.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:44.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:44.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:44.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:44.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:45.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:45.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:45.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:45.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:45.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:45.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:45.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:45.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:45.423 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:30:45.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:45.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:45.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:45.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:45.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 24613 24597 170.79 136 0.00199566 0.00584217 2024-09-13T14:30:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[103001]: pgmap v474: 99 pgs: 99 active+clean; 1002 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:30:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[103001]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:30:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3621252601' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[107556]: pgmap v474: 99 pgs: 99 active+clean; 1002 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:30:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[107556]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:30:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3621252601' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:45 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:45 smithi189 ceph-mon[90992]: pgmap v474: 99 pgs: 99 active+clean; 1002 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T14:30:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:45 smithi189 ceph-mon[90992]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:30:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3621252601' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:45 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:30:45.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:45.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:45.920 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:45.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:46.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:46.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:46.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:46.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:46.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:46.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:46.374 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:30:46.374 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:30:46.374 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:30:46.374 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:30:46.374 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:30:46.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:46.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:46.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:46.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:46.673 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 26551 26535 165.822 121.125 0.00375662 0.0060255 2024-09-13T14:30:46.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[103001]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:30:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:30:46.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:46.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:46 smithi017 ceph-mon[107556]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:30:46.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:46 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:30:46.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:46 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:30:46.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:46 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:30:46.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:46 smithi189 ceph-mon[90992]: osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:30:46.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:46.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:46.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:46.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:46.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:46.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:47.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:30:47.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:47.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:47.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:47.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:47.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:47.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:47.420 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:47.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:47.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:47.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:47.672 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 28627 28611 162.542 129.75 0.00536924 0.00614633 2024-09-13T14:30:47.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:47.679 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:47] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-13T14:30:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:47 smithi017 ceph-mon[103001]: pgmap v476: 99 pgs: 99 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.84k op/s 2024-09-13T14:30:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:47 smithi017 ceph-mon[107556]: pgmap v476: 99 pgs: 99 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.84k op/s 2024-09-13T14:30:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:47 smithi189 ceph-mon[90992]: pgmap v476: 99 pgs: 99 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.84k op/s 2024-09-13T14:30:47.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:47.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:47.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:47.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:47.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:48.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:48.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:48.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:48.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:48.269 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":474,"stamp":"2024-09-13T14:30:43.829897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1050798451,"num_objects":15992,"num_object_clones":0,"num_object_copies":47976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15992,"num_whiteouts":0,"num_read":6025,"num_read_kb":13079,"num_write":16886,"num_write_kb":1034531,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18097,"ondisk_log_size":18097,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4691652,"kb_used_data":2777412,"kb_used_omap":35,"kb_used_meta":1914076,"kb_avail":745269564,"statfs":{"total":767960285184,"available":763156033536,"internally_reserved":0,"allocated":2844069888,"data_stored":2836561935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":1960014841},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1046061899,"num_objects":15964,"num_object_clones":0,"num_object_copies":47892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15964,"num_whiteouts":0,"num_read":-242,"num_read_kb":-614,"num_write":15843,"num_write_kb":1020428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":-745717,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.941729"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":220,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.679027+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:30:36.679027+0000","last_peered":"2024-09-13T14:30:36.679027+0000","last_clean":"2024-09-13T14:30:36.679027+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:30:36.679027+0000","last_undegraded":"2024-09-13T14:30:36.679027+0000","last_fullsized":"2024-09-13T14:30:36.679027+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938611+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:30:35.938611+0000","last_peered":"2024-09-13T14:30:35.938611+0000","last_clean":"2024-09-13T14:30:35.938611+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:30:35.938611+0000","last_undegraded":"2024-09-13T14:30:35.938611+0000","last_fullsized":"2024-09-13T14:30:35.938611+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":140,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747972+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:30:35.747972+0000","last_peered":"2024-09-13T14:30:35.747972+0000","last_clean":"2024-09-13T14:30:35.747972+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:30:35.747972+0000","last_undegraded":"2024-09-13T14:30:35.747972+0000","last_fullsized":"2024-09-13T14:30:35.747972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":216,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.674975+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:30:36.674975+0000","last_peered":"2024-09-13T14:30:36.674975+0000","last_clean":"2024-09-13T14:30:36.674975+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:30:36.674975+0000","last_undegraded":"2024-09-13T14:30:36.674975+0000","last_fullsized":"2024-09-13T14:30:36.674975+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":217,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.677009+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:30:36.677009+0000","last_peered":"2024-09-13T14:30:36.677009+0000","last_clean":"2024-09-13T14:30:36.677009+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:30:36.677009+0000","last_undegraded":"2024-09-13T14:30:36.677009+0000","last_fullsized":"2024-09-13T14:30:36.677009+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":124,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939343+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:30:35.939343+0000","last_peered":"2024-09-13T14:30:35.939343+0000","last_clean":"2024-09-13T14:30:35.939343+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:30:35.939343+0000","last_undegraded":"2024-09-13T14:30:35.939343+0000","last_fullsized":"2024-09-13T14:30:35.939343+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":220,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678957+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:30:36.678957+0000","last_peered":"2024-09-13T14:30:36.678957+0000","last_clean":"2024-09-13T14:30:36.678957+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:30:36.678957+0000","last_undegraded":"2024-09-13T14:30:36.678957+0000","last_fullsized":"2024-09-13T14:30:36.678957+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":149,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156464+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:30:35.156464+0000","last_peered":"2024-09-13T14:30:35.156464+0000","last_clean":"2024-09-13T14:30:35.156464+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:30:35.156464+0000","last_undegraded":"2024-09-13T14:30:35.156464+0000","last_fullsized":"2024-09-13T14:30:35.156464+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"177'1232","reported_seq":1244,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.953716+0000","last_change":"2024-09-13T14:30:27.500678+0000","last_active":"2024-09-13T14:30:42.953716+0000","last_peered":"2024-09-13T14:30:42.953716+0000","last_clean":"2024-09-13T14:30:42.953716+0000","last_became_active":"2024-09-13T14:30:27.500235+0000","last_became_peered":"2024-09-13T14:30:27.500235+0000","last_unstale":"2024-09-13T14:30:42.953716+0000","last_undegraded":"2024-09-13T14:30:42.953716+0000","last_fullsized":"2024-09-13T14:30:42.953716+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1232,"log_dups_size":0,"ondisk_log_size":1232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:11.965913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80740352,"num_objects":1232,"num_object_clones":0,"num_object_copies":3696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1232,"num_write_kb":78848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938429+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:30:35.938429+0000","last_peered":"2024-09-13T14:30:35.938429+0000","last_clean":"2024-09-13T14:30:35.938429+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:30:35.938429+0000","last_undegraded":"2024-09-13T14:30:35.938429+0000","last_fullsized":"2024-09-13T14:30:35.938429+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747752+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:30:35.747752+0000","last_peered":"2024-09-13T14:30:35.747752+0000","last_clean":"2024-09-13T14:30:35.747752+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:30:35.747752+0000","last_undegraded":"2024-09-13T14:30:35.747752+0000","last_fullsized":"2024-09-13T14:30:35.747752+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"177'446","reported_seq":465,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:38.872126+0000","last_change":"2024-09-13T14:30:27.502589+0000","last_active":"2024-09-13T14:30:38.872126+0000","last_peered":"2024-09-13T14:30:38.872126+0000","last_clean":"2024-09-13T14:30:38.872126+0000","last_became_active":"2024-09-13T14:30:27.502327+0000","last_became_peered":"2024-09-13T14:30:27.502327+0000","last_unstale":"2024-09-13T14:30:38.872126+0000","last_undegraded":"2024-09-13T14:30:38.872126+0000","last_fullsized":"2024-09-13T14:30:38.872126+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":446,"log_dups_size":0,"ondisk_log_size":446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:12.990070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":446,"num_write_kb":28544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938987+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:30:35.938987+0000","last_peered":"2024-09-13T14:30:35.938987+0000","last_clean":"2024-09-13T14:30:35.938987+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:30:35.938987+0000","last_undegraded":"2024-09-13T14:30:35.938987+0000","last_fullsized":"2024-09-13T14:30:35.938987+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938318+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:30:35.938318+0000","last_peered":"2024-09-13T14:30:35.938318+0000","last_clean":"2024-09-13T14:30:35.938318+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:30:35.938318+0000","last_undegraded":"2024-09-13T14:30:35.938318+0000","last_fullsized":"2024-09-13T14:30:35.938318+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"177'1177","reported_seq":1189,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.945596+0000","last_change":"2024-09-13T14:30:27.500921+0000","last_active":"2024-09-13T14:30:42.945596+0000","last_peered":"2024-09-13T14:30:42.945596+0000","last_clean":"2024-09-13T14:30:42.945596+0000","last_became_active":"2024-09-13T14:30:27.500457+0000","last_became_peered":"2024-09-13T14:30:27.500457+0000","last_unstale":"2024-09-13T14:30:42.945596+0000","last_undegraded":"2024-09-13T14:30:42.945596+0000","last_fullsized":"2024-09-13T14:30:42.945596+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1177,"log_dups_size":0,"ondisk_log_size":1177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:09.832130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77135872,"num_objects":1177,"num_object_clones":0,"num_object_copies":3531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1177,"num_write_kb":75328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":125,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939827+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:30:35.939827+0000","last_peered":"2024-09-13T14:30:35.939827+0000","last_clean":"2024-09-13T14:30:35.939827+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:30:35.939827+0000","last_undegraded":"2024-09-13T14:30:35.939827+0000","last_fullsized":"2024-09-13T14:30:35.939827+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":140,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156623+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:30:35.156623+0000","last_peered":"2024-09-13T14:30:35.156623+0000","last_clean":"2024-09-13T14:30:35.156623+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:30:35.156623+0000","last_undegraded":"2024-09-13T14:30:35.156623+0000","last_fullsized":"2024-09-13T14:30:35.156623+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"177'1078","reported_seq":1097,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.411405+0000","last_change":"2024-09-13T14:30:27.501287+0000","last_active":"2024-09-13T14:30:42.411405+0000","last_peered":"2024-09-13T14:30:42.411405+0000","last_clean":"2024-09-13T14:30:42.411405+0000","last_became_active":"2024-09-13T14:30:27.500947+0000","last_became_peered":"2024-09-13T14:30:27.500947+0000","last_unstale":"2024-09-13T14:30:42.411405+0000","last_undegraded":"2024-09-13T14:30:42.411405+0000","last_fullsized":"2024-09-13T14:30:42.411405+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1078,"log_dups_size":0,"ondisk_log_size":1078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:44:23.541886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70647808,"num_objects":1078,"num_object_clones":0,"num_object_copies":3234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1078,"num_write_kb":68992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747970+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:30:35.747970+0000","last_peered":"2024-09-13T14:30:35.747970+0000","last_clean":"2024-09-13T14:30:35.747970+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:30:35.747970+0000","last_undegraded":"2024-09-13T14:30:35.747970+0000","last_fullsized":"2024-09-13T14:30:35.747970+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":218,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676248+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:30:36.676248+0000","last_peered":"2024-09-13T14:30:36.676248+0000","last_clean":"2024-09-13T14:30:36.676248+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:30:36.676248+0000","last_undegraded":"2024-09-13T14:30:36.676248+0000","last_fullsized":"2024-09-13T14:30:36.676248+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"177'1147","reported_seq":1158,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.983010+0000","last_change":"2024-09-13T14:30:27.501830+0000","last_active":"2024-09-13T14:30:42.983010+0000","last_peered":"2024-09-13T14:30:42.983010+0000","last_clean":"2024-09-13T14:30:42.983010+0000","last_became_active":"2024-09-13T14:30:27.501398+0000","last_became_peered":"2024-09-13T14:30:27.501398+0000","last_unstale":"2024-09-13T14:30:42.983010+0000","last_undegraded":"2024-09-13T14:30:42.983010+0000","last_fullsized":"2024-09-13T14:30:42.983010+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1147,"log_dups_size":0,"ondisk_log_size":1147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:52.275184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75169792,"num_objects":1147,"num_object_clones":0,"num_object_copies":3441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1147,"num_write_kb":73408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":288,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:41.431849+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:30:41.431849+0000","last_peered":"2024-09-13T14:30:41.431849+0000","last_clean":"2024-09-13T14:30:41.431849+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:30:41.431849+0000","last_undegraded":"2024-09-13T14:30:41.431849+0000","last_fullsized":"2024-09-13T14:30:41.431849+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":218,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678088+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:30:36.678088+0000","last_peered":"2024-09-13T14:30:36.678088+0000","last_clean":"2024-09-13T14:30:36.678088+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:30:36.678088+0000","last_undegraded":"2024-09-13T14:30:36.678088+0000","last_fullsized":"2024-09-13T14:30:36.678088+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938660+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:30:35.938660+0000","last_peered":"2024-09-13T14:30:35.938660+0000","last_clean":"2024-09-13T14:30:35.938660+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:30:35.938660+0000","last_undegraded":"2024-09-13T14:30:35.938660+0000","last_fullsized":"2024-09-13T14:30:35.938660+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"177'899","reported_seq":917,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:40.985352+0000","last_change":"2024-09-13T14:30:27.505193+0000","last_active":"2024-09-13T14:30:40.985352+0000","last_peered":"2024-09-13T14:30:40.985352+0000","last_clean":"2024-09-13T14:30:40.985352+0000","last_became_active":"2024-09-13T14:30:27.504909+0000","last_became_peered":"2024-09-13T14:30:27.504909+0000","last_unstale":"2024-09-13T14:30:40.985352+0000","last_undegraded":"2024-09-13T14:30:40.985352+0000","last_fullsized":"2024-09-13T14:30:40.985352+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":899,"log_dups_size":0,"ondisk_log_size":899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:40.244032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58916864,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":899,"num_write_kb":57536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":35,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.700029+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:30:42.700029+0000","last_peered":"2024-09-13T14:30:42.700029+0000","last_clean":"2024-09-13T14:30:42.700029+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:30:42.700029+0000","last_undegraded":"2024-09-13T14:30:42.700029+0000","last_fullsized":"2024-09-13T14:30:42.700029+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156563+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:30:35.156563+0000","last_peered":"2024-09-13T14:30:35.156563+0000","last_clean":"2024-09-13T14:30:35.156563+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:30:35.156563+0000","last_undegraded":"2024-09-13T14:30:35.156563+0000","last_fullsized":"2024-09-13T14:30:35.156563+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939492+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:30:35.939492+0000","last_peered":"2024-09-13T14:30:35.939492+0000","last_clean":"2024-09-13T14:30:35.939492+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:30:35.939492+0000","last_undegraded":"2024-09-13T14:30:35.939492+0000","last_fullsized":"2024-09-13T14:30:35.939492+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"177'456","reported_seq":473,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:38.868453+0000","last_change":"2024-09-13T14:30:27.503642+0000","last_active":"2024-09-13T14:30:38.868453+0000","last_peered":"2024-09-13T14:30:38.868453+0000","last_clean":"2024-09-13T14:30:38.868453+0000","last_became_active":"2024-09-13T14:30:27.503347+0000","last_became_peered":"2024-09-13T14:30:27.503347+0000","last_unstale":"2024-09-13T14:30:38.868453+0000","last_undegraded":"2024-09-13T14:30:38.868453+0000","last_fullsized":"2024-09-13T14:30:38.868453+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":456,"log_dups_size":0,"ondisk_log_size":456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:40.203221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29884416,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":456,"num_write_kb":29184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938324+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:30:35.938324+0000","last_peered":"2024-09-13T14:30:35.938324+0000","last_clean":"2024-09-13T14:30:35.938324+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:30:35.938324+0000","last_undegraded":"2024-09-13T14:30:35.938324+0000","last_fullsized":"2024-09-13T14:30:35.938324+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938987+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:30:35.938987+0000","last_peered":"2024-09-13T14:30:35.938987+0000","last_clean":"2024-09-13T14:30:35.938987+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:30:35.938987+0000","last_undegraded":"2024-09-13T14:30:35.938987+0000","last_fullsized":"2024-09-13T14:30:35.938987+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"177'1138","reported_seq":1150,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.955594+0000","last_change":"2024-09-13T14:30:27.499796+0000","last_active":"2024-09-13T14:30:42.955594+0000","last_peered":"2024-09-13T14:30:42.955594+0000","last_clean":"2024-09-13T14:30:42.955594+0000","last_became_active":"2024-09-13T14:30:27.499475+0000","last_became_peered":"2024-09-13T14:30:27.499475+0000","last_unstale":"2024-09-13T14:30:42.955594+0000","last_undegraded":"2024-09-13T14:30:42.955594+0000","last_fullsized":"2024-09-13T14:30:42.955594+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1138,"log_dups_size":0,"ondisk_log_size":1138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:07.838538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74579968,"num_objects":1138,"num_object_clones":0,"num_object_copies":3414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1138,"num_write_kb":72832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":28,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:39.679712+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:30:39.679712+0000","last_peered":"2024-09-13T14:30:39.679712+0000","last_clean":"2024-09-13T14:30:39.679712+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:30:39.679712+0000","last_undegraded":"2024-09-13T14:30:39.679712+0000","last_fullsized":"2024-09-13T14:30:39.679712+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":222,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.675027+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:30:36.675027+0000","last_peered":"2024-09-13T14:30:36.675027+0000","last_clean":"2024-09-13T14:30:36.675027+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:30:36.675027+0000","last_undegraded":"2024-09-13T14:30:36.675027+0000","last_fullsized":"2024-09-13T14:30:36.675027+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":207,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893936+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:30:35.893936+0000","last_peered":"2024-09-13T14:30:35.893936+0000","last_clean":"2024-09-13T14:30:35.893936+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:30:35.893936+0000","last_undegraded":"2024-09-13T14:30:35.893936+0000","last_fullsized":"2024-09-13T14:30:35.893936+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"177'1211","reported_seq":1223,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.952252+0000","last_change":"2024-09-13T14:30:27.505472+0000","last_active":"2024-09-13T14:30:42.952252+0000","last_peered":"2024-09-13T14:30:42.952252+0000","last_clean":"2024-09-13T14:30:42.952252+0000","last_became_active":"2024-09-13T14:30:27.505098+0000","last_became_peered":"2024-09-13T14:30:27.505098+0000","last_unstale":"2024-09-13T14:30:42.952252+0000","last_undegraded":"2024-09-13T14:30:42.952252+0000","last_fullsized":"2024-09-13T14:30:42.952252+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1211,"log_dups_size":0,"ondisk_log_size":1211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:45.218708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79364096,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1211,"num_write_kb":77504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":279,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939392+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:30:35.939392+0000","last_peered":"2024-09-13T14:30:35.939392+0000","last_clean":"2024-09-13T14:30:35.939392+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:30:35.939392+0000","last_undegraded":"2024-09-13T14:30:35.939392+0000","last_fullsized":"2024-09-13T14:30:35.939392+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":742,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747796+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:30:35.747796+0000","last_peered":"2024-09-13T14:30:35.747796+0000","last_clean":"2024-09-13T14:30:35.747796+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:30:35.747796+0000","last_undegraded":"2024-09-13T14:30:35.747796+0000","last_fullsized":"2024-09-13T14:30:35.747796+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.677747+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:30:36.677747+0000","last_peered":"2024-09-13T14:30:36.677747+0000","last_clean":"2024-09-13T14:30:36.677747+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:30:36.677747+0000","last_undegraded":"2024-09-13T14:30:36.677747+0000","last_fullsized":"2024-09-13T14:30:36.677747+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":123,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156758+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:30:35.156758+0000","last_peered":"2024-09-13T14:30:35.156758+0000","last_clean":"2024-09-13T14:30:35.156758+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:30:35.156758+0000","last_undegraded":"2024-09-13T14:30:35.156758+0000","last_fullsized":"2024-09-13T14:30:35.156758+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"177'1097","reported_seq":1116,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.222260+0000","last_change":"2024-09-13T14:30:27.505328+0000","last_active":"2024-09-13T14:30:42.222260+0000","last_peered":"2024-09-13T14:30:42.222260+0000","last_clean":"2024-09-13T14:30:42.222260+0000","last_became_active":"2024-09-13T14:30:27.505015+0000","last_became_peered":"2024-09-13T14:30:27.505015+0000","last_unstale":"2024-09-13T14:30:42.222260+0000","last_undegraded":"2024-09-13T14:30:42.222260+0000","last_fullsized":"2024-09-13T14:30:42.222260+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1097,"log_dups_size":0,"ondisk_log_size":1097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:42.013686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1097,"num_write_kb":70208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":51,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:39.822827+0000","last_change":"2024-09-13T14:30:39.822827+0000","last_active":"2024-09-13T14:30:39.822827+0000","last_peered":"2024-09-13T14:30:39.822827+0000","last_clean":"2024-09-13T14:30:39.822827+0000","last_became_active":"2024-09-13T14:30:11.437578+0000","last_became_peered":"2024-09-13T14:30:11.437578+0000","last_unstale":"2024-09-13T14:30:39.822827+0000","last_undegraded":"2024-09-13T14:30:39.822827+0000","last_fullsized":"2024-09-13T14:30:39.822827+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.822747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.822747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:03.728547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":550,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939279+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:30:35.939279+0000","last_peered":"2024-09-13T14:30:35.939279+0000","last_clean":"2024-09-13T14:30:35.939279+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:30:35.939279+0000","last_undegraded":"2024-09-13T14:30:35.939279+0000","last_fullsized":"2024-09-13T14:30:35.939279+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":217,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.675196+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:30:36.675196+0000","last_peered":"2024-09-13T14:30:36.675196+0000","last_clean":"2024-09-13T14:30:36.675196+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:30:36.675196+0000","last_undegraded":"2024-09-13T14:30:36.675196+0000","last_fullsized":"2024-09-13T14:30:36.675196+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":229,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.677193+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:30:36.677193+0000","last_peered":"2024-09-13T14:30:36.677193+0000","last_clean":"2024-09-13T14:30:36.677193+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:30:36.677193+0000","last_undegraded":"2024-09-13T14:30:36.677193+0000","last_fullsized":"2024-09-13T14:30:36.677193+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"177'1204","reported_seq":1216,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.949221+0000","last_change":"2024-09-13T14:30:27.500479+0000","last_active":"2024-09-13T14:30:42.949221+0000","last_peered":"2024-09-13T14:30:42.949221+0000","last_clean":"2024-09-13T14:30:42.949221+0000","last_became_active":"2024-09-13T14:30:27.500110+0000","last_became_peered":"2024-09-13T14:30:27.500110+0000","last_unstale":"2024-09-13T14:30:42.949221+0000","last_undegraded":"2024-09-13T14:30:42.949221+0000","last_fullsized":"2024-09-13T14:30:42.949221+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1204,"log_dups_size":0,"ondisk_log_size":1204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:10.174744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78905344,"num_objects":1204,"num_object_clones":0,"num_object_copies":3612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1204,"num_write_kb":77056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":288,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:37.362025+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:30:37.362025+0000","last_peered":"2024-09-13T14:30:37.362025+0000","last_clean":"2024-09-13T14:30:37.362025+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:30:37.362025+0000","last_undegraded":"2024-09-13T14:30:37.362025+0000","last_fullsized":"2024-09-13T14:30:37.362025+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":646,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.675442+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:30:36.675442+0000","last_peered":"2024-09-13T14:30:36.675442+0000","last_clean":"2024-09-13T14:30:36.675442+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:30:36.675442+0000","last_undegraded":"2024-09-13T14:30:36.675442+0000","last_fullsized":"2024-09-13T14:30:36.675442+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":207,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893399+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:30:35.893399+0000","last_peered":"2024-09-13T14:30:35.893399+0000","last_clean":"2024-09-13T14:30:35.893399+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:30:35.893399+0000","last_undegraded":"2024-09-13T14:30:35.893399+0000","last_fullsized":"2024-09-13T14:30:35.893399+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":223,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.675957+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:30:36.675957+0000","last_peered":"2024-09-13T14:30:36.675957+0000","last_clean":"2024-09-13T14:30:36.675957+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:30:36.675957+0000","last_undegraded":"2024-09-13T14:30:36.675957+0000","last_fullsized":"2024-09-13T14:30:36.675957+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"177'464","reported_seq":483,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:38.868159+0000","last_change":"2024-09-13T14:30:27.501260+0000","last_active":"2024-09-13T14:30:38.868159+0000","last_peered":"2024-09-13T14:30:38.868159+0000","last_clean":"2024-09-13T14:30:38.868159+0000","last_became_active":"2024-09-13T14:30:27.500950+0000","last_became_peered":"2024-09-13T14:30:27.500950+0000","last_unstale":"2024-09-13T14:30:38.868159+0000","last_undegraded":"2024-09-13T14:30:38.868159+0000","last_fullsized":"2024-09-13T14:30:38.868159+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":464,"log_dups_size":0,"ondisk_log_size":464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:54.878021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":464,"num_write_kb":29696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":28,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:40.695970+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:30:40.695970+0000","last_peered":"2024-09-13T14:30:40.695970+0000","last_clean":"2024-09-13T14:30:40.695970+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:30:40.695970+0000","last_undegraded":"2024-09-13T14:30:40.695970+0000","last_fullsized":"2024-09-13T14:30:40.695970+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":791,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.748088+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:30:35.748088+0000","last_peered":"2024-09-13T14:30:35.748088+0000","last_clean":"2024-09-13T14:30:35.748088+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:30:35.748088+0000","last_undegraded":"2024-09-13T14:30:35.748088+0000","last_fullsized":"2024-09-13T14:30:35.748088+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":142,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.937979+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:30:35.937979+0000","last_peered":"2024-09-13T14:30:35.937979+0000","last_clean":"2024-09-13T14:30:35.937979+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:30:35.937979+0000","last_undegraded":"2024-09-13T14:30:35.937979+0000","last_fullsized":"2024-09-13T14:30:35.937979+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893715+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:30:35.893715+0000","last_peered":"2024-09-13T14:30:35.893715+0000","last_clean":"2024-09-13T14:30:35.893715+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:30:35.893715+0000","last_undegraded":"2024-09-13T14:30:35.893715+0000","last_fullsized":"2024-09-13T14:30:35.893715+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"177'1061","reported_seq":1080,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.411120+0000","last_change":"2024-09-13T14:30:27.502045+0000","last_active":"2024-09-13T14:30:42.411120+0000","last_peered":"2024-09-13T14:30:42.411120+0000","last_clean":"2024-09-13T14:30:42.411120+0000","last_became_active":"2024-09-13T14:30:27.501711+0000","last_became_peered":"2024-09-13T14:30:27.501711+0000","last_unstale":"2024-09-13T14:30:42.411120+0000","last_undegraded":"2024-09-13T14:30:42.411120+0000","last_fullsized":"2024-09-13T14:30:42.411120+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1061,"log_dups_size":0,"ondisk_log_size":1061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:21.043630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69533696,"num_objects":1061,"num_object_clones":0,"num_object_copies":3183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1061,"num_write_kb":67904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":176,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:37.430258+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:30:37.430258+0000","last_peered":"2024-09-13T14:30:37.430258+0000","last_clean":"2024-09-13T14:30:37.430258+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:30:37.430258+0000","last_undegraded":"2024-09-13T14:30:37.430258+0000","last_fullsized":"2024-09-13T14:30:37.430258+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.748047+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:30:35.748047+0000","last_peered":"2024-09-13T14:30:35.748047+0000","last_clean":"2024-09-13T14:30:35.748047+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:30:35.748047+0000","last_undegraded":"2024-09-13T14:30:35.748047+0000","last_fullsized":"2024-09-13T14:30:35.748047+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":817,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893854+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:30:35.893854+0000","last_peered":"2024-09-13T14:30:35.893854+0000","last_clean":"2024-09-13T14:30:35.893854+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:30:35.893854+0000","last_undegraded":"2024-09-13T14:30:35.893854+0000","last_fullsized":"2024-09-13T14:30:35.893854+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":196,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893587+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:30:35.893587+0000","last_peered":"2024-09-13T14:30:35.893587+0000","last_clean":"2024-09-13T14:30:35.893587+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:30:35.893587+0000","last_undegraded":"2024-09-13T14:30:35.893587+0000","last_fullsized":"2024-09-13T14:30:35.893587+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"177'1030","reported_seq":1049,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.227691+0000","last_change":"2024-09-13T14:30:27.500989+0000","last_active":"2024-09-13T14:30:42.227691+0000","last_peered":"2024-09-13T14:30:42.227691+0000","last_clean":"2024-09-13T14:30:42.227691+0000","last_became_active":"2024-09-13T14:30:27.500671+0000","last_became_peered":"2024-09-13T14:30:27.500671+0000","last_unstale":"2024-09-13T14:30:42.227691+0000","last_undegraded":"2024-09-13T14:30:42.227691+0000","last_fullsized":"2024-09-13T14:30:42.227691+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1030,"log_dups_size":0,"ondisk_log_size":1030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:59.179207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67502080,"num_objects":1030,"num_object_clones":0,"num_object_copies":3090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1030,"num_write_kb":65920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":176,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:38.471219+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:30:38.471219+0000","last_peered":"2024-09-13T14:30:38.471219+0000","last_clean":"2024-09-13T14:30:38.471219+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:30:38.471219+0000","last_undegraded":"2024-09-13T14:30:38.471219+0000","last_fullsized":"2024-09-13T14:30:38.471219+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":207,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893760+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:30:35.893760+0000","last_peered":"2024-09-13T14:30:35.893760+0000","last_clean":"2024-09-13T14:30:35.893760+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:30:35.893760+0000","last_undegraded":"2024-09-13T14:30:35.893760+0000","last_fullsized":"2024-09-13T14:30:35.893760+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":487,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939512+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:30:35.939512+0000","last_peered":"2024-09-13T14:30:35.939512+0000","last_clean":"2024-09-13T14:30:35.939512+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:30:35.939512+0000","last_undegraded":"2024-09-13T14:30:35.939512+0000","last_fullsized":"2024-09-13T14:30:35.939512+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676116+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:30:36.676116+0000","last_peered":"2024-09-13T14:30:36.676116+0000","last_clean":"2024-09-13T14:30:36.676116+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:30:36.676116+0000","last_undegraded":"2024-09-13T14:30:36.676116+0000","last_fullsized":"2024-09-13T14:30:36.676116+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"177'1181","reported_seq":1193,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.962813+0000","last_change":"2024-09-13T14:30:27.502612+0000","last_active":"2024-09-13T14:30:42.962813+0000","last_peered":"2024-09-13T14:30:42.962813+0000","last_clean":"2024-09-13T14:30:42.962813+0000","last_became_active":"2024-09-13T14:30:27.501659+0000","last_became_peered":"2024-09-13T14:30:27.501659+0000","last_unstale":"2024-09-13T14:30:42.962813+0000","last_undegraded":"2024-09-13T14:30:42.962813+0000","last_fullsized":"2024-09-13T14:30:42.962813+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1181,"log_dups_size":0,"ondisk_log_size":1181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:47.542650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1181,"num_write_kb":75584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":28,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:37.305705+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:30:37.305705+0000","last_peered":"2024-09-13T14:30:37.305705+0000","last_clean":"2024-09-13T14:30:37.305705+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:30:37.305705+0000","last_undegraded":"2024-09-13T14:30:37.305705+0000","last_fullsized":"2024-09-13T14:30:37.305705+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":219,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.679107+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:30:36.679107+0000","last_peered":"2024-09-13T14:30:36.679107+0000","last_clean":"2024-09-13T14:30:36.679107+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:30:36.679107+0000","last_undegraded":"2024-09-13T14:30:36.679107+0000","last_fullsized":"2024-09-13T14:30:36.679107+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":551,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893984+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:30:35.893984+0000","last_peered":"2024-09-13T14:30:35.893984+0000","last_clean":"2024-09-13T14:30:35.893984+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:30:35.893984+0000","last_undegraded":"2024-09-13T14:30:35.893984+0000","last_fullsized":"2024-09-13T14:30:35.893984+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.674788+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:30:36.674788+0000","last_peered":"2024-09-13T14:30:36.674788+0000","last_clean":"2024-09-13T14:30:36.674788+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:30:36.674788+0000","last_undegraded":"2024-09-13T14:30:36.674788+0000","last_fullsized":"2024-09-13T14:30:36.674788+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"177'1144","reported_seq":1156,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:42.936392+0000","last_change":"2024-09-13T14:30:27.501009+0000","last_active":"2024-09-13T14:30:42.936392+0000","last_peered":"2024-09-13T14:30:42.936392+0000","last_clean":"2024-09-13T14:30:42.936392+0000","last_became_active":"2024-09-13T14:30:27.500481+0000","last_became_peered":"2024-09-13T14:30:27.500481+0000","last_unstale":"2024-09-13T14:30:42.936392+0000","last_undegraded":"2024-09-13T14:30:42.936392+0000","last_fullsized":"2024-09-13T14:30:42.936392+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":1144,"log_dups_size":0,"ondisk_log_size":1144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:30.536488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1144,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":48,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678851+0000","last_change":"2024-09-13T14:30:35.612918+0000","last_active":"2024-09-13T14:30:36.678851+0000","last_peered":"2024-09-13T14:30:36.678851+0000","last_clean":"2024-09-13T14:30:36.678851+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:30:36.678851+0000","last_undegraded":"2024-09-13T14:30:36.678851+0000","last_fullsized":"2024-09-13T14:30:36.678851+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:09:56.715529+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T14:30:39.644446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":221,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676903+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:30:36.676903+0000","last_peered":"2024-09-13T14:30:36.676903+0000","last_clean":"2024-09-13T14:30:36.676903+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:30:36.676903+0000","last_undegraded":"2024-09-13T14:30:36.676903+0000","last_fullsized":"2024-09-13T14:30:36.676903+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":595,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156804+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:30:35.156804+0000","last_peered":"2024-09-13T14:30:35.156804+0000","last_clean":"2024-09-13T14:30:35.156804+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:30:35.156804+0000","last_undegraded":"2024-09-13T14:30:35.156804+0000","last_fullsized":"2024-09-13T14:30:35.156804+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":124,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938246+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:30:35.938246+0000","last_peered":"2024-09-13T14:30:35.938246+0000","last_clean":"2024-09-13T14:30:35.938246+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:30:35.938246+0000","last_undegraded":"2024-09-13T14:30:35.938246+0000","last_fullsized":"2024-09-13T14:30:35.938246+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939170+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:30:35.939170+0000","last_peered":"2024-09-13T14:30:35.939170+0000","last_clean":"2024-09-13T14:30:35.939170+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:30:35.939170+0000","last_undegraded":"2024-09-13T14:30:35.939170+0000","last_fullsized":"2024-09-13T14:30:35.939170+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":147,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938738+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:30:35.938738+0000","last_peered":"2024-09-13T14:30:35.938738+0000","last_clean":"2024-09-13T14:30:35.938738+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:30:35.938738+0000","last_undegraded":"2024-09-13T14:30:35.938738+0000","last_fullsized":"2024-09-13T14:30:35.938738+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":207,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893498+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:30:35.893498+0000","last_peered":"2024-09-13T14:30:35.893498+0000","last_clean":"2024-09-13T14:30:35.893498+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:30:35.893498+0000","last_undegraded":"2024-09-13T14:30:35.893498+0000","last_fullsized":"2024-09-13T14:30:35.893498+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":144,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.939751+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:30:35.939751+0000","last_peered":"2024-09-13T14:30:35.939751+0000","last_clean":"2024-09-13T14:30:35.939751+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:30:35.939751+0000","last_undegraded":"2024-09-13T14:30:35.939751+0000","last_fullsized":"2024-09-13T14:30:35.939751+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747839+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:30:35.747839+0000","last_peered":"2024-09-13T14:30:35.747839+0000","last_clean":"2024-09-13T14:30:35.747839+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:30:35.747839+0000","last_undegraded":"2024-09-13T14:30:35.747839+0000","last_fullsized":"2024-09-13T14:30:35.747839+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.677747+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:30:36.677747+0000","last_peered":"2024-09-13T14:30:36.677747+0000","last_clean":"2024-09-13T14:30:36.677747+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:30:36.677747+0000","last_undegraded":"2024-09-13T14:30:36.677747+0000","last_fullsized":"2024-09-13T14:30:36.677747+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676247+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:30:36.676247+0000","last_peered":"2024-09-13T14:30:36.676247+0000","last_clean":"2024-09-13T14:30:36.676247+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:30:36.676247+0000","last_undegraded":"2024-09-13T14:30:36.676247+0000","last_fullsized":"2024-09-13T14:30:36.676247+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.674691+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:30:36.674691+0000","last_peered":"2024-09-13T14:30:36.674691+0000","last_clean":"2024-09-13T14:30:36.674691+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:30:36.674691+0000","last_undegraded":"2024-09-13T14:30:36.674691+0000","last_fullsized":"2024-09-13T14:30:36.674691+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":140,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.156761+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:30:35.156761+0000","last_peered":"2024-09-13T14:30:35.156761+0000","last_clean":"2024-09-13T14:30:35.156761+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:30:35.156761+0000","last_undegraded":"2024-09-13T14:30:35.156761+0000","last_fullsized":"2024-09-13T14:30:35.156761+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":238,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.674933+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:30:36.674933+0000","last_peered":"2024-09-13T14:30:36.674933+0000","last_clean":"2024-09-13T14:30:36.674933+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:30:36.674933+0000","last_undegraded":"2024-09-13T14:30:36.674933+0000","last_fullsized":"2024-09-13T14:30:36.674933+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.677097+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:30:36.677097+0000","last_peered":"2024-09-13T14:30:36.677097+0000","last_clean":"2024-09-13T14:30:36.677097+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:30:36.677097+0000","last_undegraded":"2024-09-13T14:30:36.677097+0000","last_fullsized":"2024-09-13T14:30:36.677097+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.893651+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:30:35.893651+0000","last_peered":"2024-09-13T14:30:35.893651+0000","last_clean":"2024-09-13T14:30:35.893651+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:30:35.893651+0000","last_undegraded":"2024-09-13T14:30:35.893651+0000","last_fullsized":"2024-09-13T14:30:35.893651+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678612+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:30:36.678612+0000","last_peered":"2024-09-13T14:30:36.678612+0000","last_clean":"2024-09-13T14:30:36.678612+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:30:36.678612+0000","last_undegraded":"2024-09-13T14:30:36.678612+0000","last_fullsized":"2024-09-13T14:30:36.678612+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":160,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938529+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:30:35.938529+0000","last_peered":"2024-09-13T14:30:35.938529+0000","last_clean":"2024-09-13T14:30:35.938529+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:30:35.938529+0000","last_undegraded":"2024-09-13T14:30:35.938529+0000","last_fullsized":"2024-09-13T14:30:35.938529+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":240,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.675838+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:30:36.675838+0000","last_peered":"2024-09-13T14:30:36.675838+0000","last_clean":"2024-09-13T14:30:36.675838+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:30:36.675838+0000","last_undegraded":"2024-09-13T14:30:36.675838+0000","last_fullsized":"2024-09-13T14:30:36.675838+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":209,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676004+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:30:36.676004+0000","last_peered":"2024-09-13T14:30:36.676004+0000","last_clean":"2024-09-13T14:30:36.676004+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:30:36.676004+0000","last_undegraded":"2024-09-13T14:30:36.676004+0000","last_fullsized":"2024-09-13T14:30:36.676004+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":158,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938695+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:30:35.938695+0000","last_peered":"2024-09-13T14:30:35.938695+0000","last_clean":"2024-09-13T14:30:35.938695+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:30:35.938695+0000","last_undegraded":"2024-09-13T14:30:35.938695+0000","last_fullsized":"2024-09-13T14:30:35.938695+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":143,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747898+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:30:35.747898+0000","last_peered":"2024-09-13T14:30:35.747898+0000","last_clean":"2024-09-13T14:30:35.747898+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:30:35.747898+0000","last_undegraded":"2024-09-13T14:30:35.747898+0000","last_fullsized":"2024-09-13T14:30:35.747898+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":158,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938454+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:30:35.938454+0000","last_peered":"2024-09-13T14:30:35.938454+0000","last_clean":"2024-09-13T14:30:35.938454+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:30:35.938454+0000","last_undegraded":"2024-09-13T14:30:35.938454+0000","last_fullsized":"2024-09-13T14:30:35.938454+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":126,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.747993+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:30:35.747993+0000","last_peered":"2024-09-13T14:30:35.747993+0000","last_clean":"2024-09-13T14:30:35.747993+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:30:35.747993+0000","last_undegraded":"2024-09-13T14:30:35.747993+0000","last_fullsized":"2024-09-13T14:30:35.747993+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":241,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678005+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:30:36.678005+0000","last_peered":"2024-09-13T14:30:36.678005+0000","last_clean":"2024-09-13T14:30:36.678005+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:30:36.678005+0000","last_undegraded":"2024-09-13T14:30:36.678005+0000","last_fullsized":"2024-09-13T14:30:36.678005+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":227,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.678724+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:30:36.678724+0000","last_peered":"2024-09-13T14:30:36.678724+0000","last_clean":"2024-09-13T14:30:36.678724+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:30:36.678724+0000","last_undegraded":"2024-09-13T14:30:36.678724+0000","last_fullsized":"2024-09-13T14:30:36.678724+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":151,"reported_epoch":176,"state":"active+clean","last_fresh":"2024-09-13T14:30:35.938113+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:30:35.938113+0000","last_peered":"2024-09-13T14:30:35.938113+0000","last_clean":"2024-09-13T14:30:35.938113+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:30:35.938113+0000","last_undegraded":"2024-09-13T14:30:35.938113+0000","last_fullsized":"2024-09-13T14:30:35.938113+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":221,"reported_epoch":177,"state":"active+clean","last_fresh":"2024-09-13T14:30:36.676124+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:30:36.676124+0000","last_peered":"2024-09-13T14:30:36.676124+0000","last_clean":"2024-09-13T14:30:36.676124+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:30:36.676124+0000","last_undegraded":"2024-09-13T14:30:36.676124+0000","last_fullsized":"2024-09-13T14:30:36.676124+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1046282240,"num_objects":15965,"num_object_clones":0,"num_object_copies":47895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15965,"num_write_kb":1021760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2998140928,"data_stored":2998140928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15965,"ondisk_log_size":15965,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":316358,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397735,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":731224,"kb_used_data":425552,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93013928,"statfs":{"total":95995035648,"available":95246262272,"internally_reserved":0,"allocated":435765248,"data_stored":434813739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593963,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":663752,"kb_used_data":358080,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93081400,"statfs":{"total":95995035648,"available":95315353600,"internally_reserved":0,"allocated":366673920,"data_stored":365738101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790193,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671108,"kb_used_data":369596,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93074044,"statfs":{"total":95995035648,"available":95307821056,"internally_reserved":0,"allocated":378466304,"data_stored":377530888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986421,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652948,"kb_used_data":347276,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93092204,"statfs":{"total":95995035648,"available":95326416896,"internally_reserved":0,"allocated":355610624,"data_stored":354663141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791696978,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":434180,"kb_used_data":267836,"kb_used_omap":4,"kb_used_meta":166267,"kb_avail":93310972,"statfs":{"total":95995035648,"available":95550435328,"internally_reserved":0,"allocated":274264064,"data_stored":273328721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":170257483},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860505,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557192,"kb_used_data":417216,"kb_used_omap":8,"kb_used_meta":139959,"kb_avail":93187960,"statfs":{"total":95995035648,"available":95424471040,"internally_reserved":0,"allocated":427229184,"data_stored":426290267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":143318860},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024032,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":519872,"kb_used_data":322424,"kb_used_omap":7,"kb_used_meta":197432,"kb_avail":93225280,"statfs":{"total":95995035648,"available":95462686720,"internally_reserved":0,"allocated":330162176,"data_stored":329230306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":202170807},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187567,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":461376,"kb_used_data":269432,"kb_used_omap":7,"kb_used_meta":191928,"kb_avail":93283776,"statfs":{"total":95995035648,"available":95522586624,"internally_reserved":0,"allocated":275898368,"data_stored":274966772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":196534603},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":284098560,"data_stored":284098560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":341180416,"data_stored":341180416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":444203008,"data_stored":444203008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":311361536,"data_stored":311361536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":388694016,"data_stored":388694016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":389480448,"data_stored":389480448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":378863616,"data_stored":378863616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":460259328,"data_stored":460259328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:30:48.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:30:48.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:48.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:48.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:48.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:48.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:48.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:48.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 30842 30826 160.522 138.438 0.00413195 0.00622381 2024-09-13T14:30:48.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:48.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:48.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:48.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:48.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:48.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:49.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:49.057 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:49.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:49.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:49.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:49.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:49.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:49.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:49.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:49.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:49.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:49 smithi017 ceph-mon[103001]: pgmap v477: 99 pgs: 99 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2024-09-13T14:30:49.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:49 smithi017 ceph-mon[107556]: pgmap v477: 99 pgs: 99 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2024-09-13T14:30:49.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 15 32823 32808 157.7 123.875 0.00772467 0.00633501 2024-09-13T14:30:49.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:49.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:49 smithi189 ceph-mon[90992]: pgmap v477: 99 pgs: 99 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2024-09-13T14:30:49.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:49.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:49.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:49.994 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:49.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:49.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:50.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:50.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:50.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:50.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:50.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:50.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:50.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:30:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:30:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:50.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:50.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:50.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:50.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 15 34840 34825 155.438 126.062 0.00440282 0.00642803 2024-09-13T14:30:50.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:50.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:50.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:50.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:50.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.851 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:74513 5.7 5:eba6007a:::benchmark_data_smithi017_147622_object74512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e110)", 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 275.230132045, 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088206253999999998, 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 74513 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615767+0000", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615766+0000", 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615827+0000", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1118999999999995e-05 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615842+0000", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4612e-05 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615847+0000", 2024-09-13T14:30:50.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2259999999999996e-06 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.615910+0000", 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3670000000000005e-05 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.616014+0000", 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010381 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.616146+0000", 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013156299999999999 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.619203+0000", 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030568549999999998 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.619236+0000", 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3325999999999999e-05 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703883+0000", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084646937000000005 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703935+0000", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2346999999999999e-05 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:15.703974+0000", 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8306000000000002e-05 2024-09-13T14:30:50.857 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89287 5.7 5:edd860ce:::benchmark_data_smithi017_147622_object89286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 263.978559346, 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096836681999999993, 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.858 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89287 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867340+0000", 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867339+0000", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867370+0000", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0840999999999998e-05 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867379+0000", 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5699999999999999e-06 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867384+0000", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4139999999999996e-06 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867466+0000", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2606000000000002e-05 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867583+0000", 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011711599999999999 2024-09-13T14:30:50.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.867713+0000", 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013018099999999999 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871241+0000", 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035272559999999999 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876787+0000", 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055461260000000002 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964085+0000", 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087298007999999996 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964136+0000", 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0902999999999997e-05 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.964177+0000", 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0966e-05 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89292 5.7 5:e49b0fb3:::benchmark_data_smithi017_147622_object89291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 263.97708244900002, 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097664913000000006, 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.864 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.865 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89292 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868817+0000", 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868816+0000", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868853+0000", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6808999999999999e-05 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868860+0000", 2024-09-13T14:30:50.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2400000000000001e-06 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.868864+0000", 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9090000000000002e-06 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871269+0000", 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024050870000000002 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871391+0000", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012149799999999999 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871488+0000", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6978999999999994e-05 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876857+0000", 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0053692790000000002 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879226+0000", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002368906 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966399+0000", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087172629000000001 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966449+0000", 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0000000000000002e-05 2024-09-13T14:30:50.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966482+0000", 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3136999999999998e-05 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89296 5.7 5:e4b5a107:::benchmark_data_smithi017_147622_object89295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 263.97452426299998, 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.095206473, 2024-09-13T14:30:50.871 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89296 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871375+0000", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871374+0000", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871402+0000", 2024-09-13T14:30:50.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8283e-05 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871413+0000", 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0832000000000001e-05 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871418+0000", 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3029999999999999e-06 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874286+0000", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028671970000000001 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874387+0000", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101537 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.874508+0000", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121116 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879128+0000", 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046201239999999998 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879232+0000", 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010359199999999999 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966520+0000", 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087287662000000002 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966556+0000", 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5957000000000001e-05 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.966582+0000", 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5950000000000001e-05 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.877 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89297 5.7 5:eed05594:::benchmark_data_smithi017_147622_object89296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 263.974195769, 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.103594748, 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89297 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:30:50.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871704+0000", 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871703+0000", 2024-09-13T14:30:50.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871764+0000", 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1189e-05 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871775+0000", 2024-09-13T14:30:50.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0699e-05 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.871779+0000", 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7319999999999998e-06 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.876883+0000", 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0051035389999999998 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.877015+0000", 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132323 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.877156+0000", 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000140853 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879144+0000", 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019881629999999998 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879270+0000", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012565300000000001 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975170+0000", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.095899897999999997 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975260+0000", 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0172000000000003e-05 2024-09-13T14:30:50.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975298+0000", 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8241000000000002e-05 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:89303 5.7 5:ea49a0ca:::benchmark_data_smithi017_147622_object89302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 263.96652633600002, 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096065179000000001, 2024-09-13T14:30:50.884 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89303 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879373+0000", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879372+0000", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879408+0000", 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6168000000000003e-05 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879421+0000", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3124e-05 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879426+0000", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8969999999999999e-06 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879462+0000", 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6359999999999997e-05 2024-09-13T14:30:50.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879574+0000", 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111155 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.879712+0000", 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013855900000000001 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.881300+0000", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0015874369999999999 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.881428+0000", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012835799999999999 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975333+0000", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.093904986999999995 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975405+0000", 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2446000000000004e-05 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:26.975438+0000", 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2879000000000001e-05 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109098 5.f 5:f3c36ffb:::benchmark_data_smithi017_147622_object109097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:30:50.890 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.33553199900001, 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094563000999999994, 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109098 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:30:50.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510367+0000", 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510366+0000", 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510398+0000", 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1823999999999999e-05 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510410+0000", 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2128e-05 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510415+0000", 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6380000000000001e-06 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510815+0000", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000399317 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510936+0000", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121722 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.511065+0000", 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012823000000000001 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599922+0000", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088857631000000006 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599955+0000", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2291e-05 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604840+0000", 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00488551 2024-09-13T14:30:50.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604891+0000", 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0939000000000001e-05 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604930+0000", 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9393e-05 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109100 5.f 5:f32deec1:::benchmark_data_smithi017_147622_object109099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.33547476800001, 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094580336000000001, 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109100 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510425+0000", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510424+0000", 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510504+0000", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0269e-05 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510716+0000", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021181999999999999 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.510723+0000", 2024-09-13T14:30:50.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5810000000000002e-06 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515083+0000", 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043600599999999998 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515212+0000", 2024-09-13T14:30:50.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012924 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.515380+0000", 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016826100000000001 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599981+0000", 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084601126999999998 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.603595+0000", 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036140569999999999 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604941+0000", 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0013456259999999999 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.604976+0000", 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.506e-05 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605005+0000", 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8768e-05 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109105 5.f 5:fdd6df1a:::benchmark_data_smithi017_147622_object109104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.333015402, 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.092191632999999995, 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.903 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109105 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512884+0000", 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512883+0000", 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512934+0000", 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0825999999999998e-05 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512945+0000", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1219e-05 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.512950+0000", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3600000000000004e-06 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519194+0000", 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062439180000000002 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519309+0000", 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011505 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.519438+0000", 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012902899999999999 2024-09-13T14:30:50.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600004+0000", 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.080566213999999997 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.603619+0000", 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003614492 2024-09-13T14:30:50.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605011+0000", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00139258 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605046+0000", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4820000000000002e-05 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605076+0000", 2024-09-13T14:30:50.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9302000000000002e-05 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109104 5.2 e112/104 5:4e10141e:::benchmark_data_smithi017_147622_object109103:head v 112'6801, mlcod=112'6797)", 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.32339641600001, 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.096480472999999997, 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522503+0000", 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522522+0000", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9335e-05 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522528+0000", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9529999999999996e-06 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.522531+0000", 2024-09-13T14:30:50.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2219999999999998e-06 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608519+0000", 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.085988093000000002 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608537+0000", 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8139999999999999e-05 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.618668+0000", 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010130251 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.618983+0000", 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00031575400000000001 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:109109 5.f 5:f93ff6aa:::benchmark_data_smithi017_147622_object109108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:30:50.915 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.322612206, 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088054187000000006, 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 109109 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523287+0000", 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523286+0000", 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523318+0000", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2270000000000001e-05 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523329+0000", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0380999999999999e-05 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.523334+0000", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4380000000000003e-06 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600029+0000", 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.076695248999999993 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600167+0000", 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013783299999999999 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.600306+0000", 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013934099999999999 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.605082+0000", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047755430000000001 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.608456+0000", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0033741959999999999 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611258+0000", 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028014479999999998 2024-09-13T14:30:50.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611307+0000", 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8899000000000001e-05 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.611341+0000", 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4808000000000003e-05 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35034.0:109114 5.e e112/104 5:7c0f1b50:::benchmark_data_smithi017_147622_object109113:head v 112'6762, mlcod=112'6760)", 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 246.31916741000001, 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097411384000000004, 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526732+0000", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526731+0000", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.526754+0000", 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3203000000000001e-05 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599845+0000", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073090426 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599866+0000", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.128e-05 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599947+0000", 2024-09-13T14:30:50.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0652e-05 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.599969+0000", 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.228e-05 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.623992+0000", 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024023052 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:26:44.624143+0000", 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015129400000000001 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:158669 5.7 5:eaa5c984:::benchmark_data_smithi017_147622_object24040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 193.34940724399999, 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.090928470999999997, 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.926 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 158669 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496492+0000", 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496491+0000", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496495+0000", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9940000000000002e-06 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496507+0000", 2024-09-13T14:30:50.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1243000000000001e-05 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.496513+0000", 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511633+0000", 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015120188 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511737+0000", 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103869 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.511851+0000", 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011382700000000001 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.517530+0000", 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005679585 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.526651+0000", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091204649999999995 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587351+0000", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.060699837999999999 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587386+0000", 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4789000000000002e-05 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.587421+0000", 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5061999999999997e-05 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:158681 5.7 5:e5e33199:::benchmark_data_smithi017_147622_object24052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 193.33542766799999, 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.101405525, 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.932 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 158681 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510472+0000", 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510470+0000", 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510477+0000", 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1130000000000003e-06 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510489+0000", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2063e-05 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.510495+0000", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5310000000000001e-06 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.517730+0000", 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072353929999999997 2024-09-13T14:30:50.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.521211+0000", 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003480698 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.521315+0000", 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010391300000000001 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.526879+0000", 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005564083 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.527227+0000", 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034801400000000001 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611799+0000", 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084571795000000005 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611839+0000", 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0491999999999998e-05 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:27:37.611877+0000", 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8022000000000003e-05 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.938 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241036 5.16 5:6b1e2ede:::benchmark_data_smithi017_147622_object106407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:20.686620+0000", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 90.159279819000005, 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83059148800000004, 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241036 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686620+0000", 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686620+0000", 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686619+0000", 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686622+0000", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6860000000000002e-06 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686628+0000", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3010000000000003e-06 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686632+0000", 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8770000000000003e-06 2024-09-13T14:30:50.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686698+0000", 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5908000000000007e-05 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.686706+0000", 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6990000000000003e-06 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.510955+0000", 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.82424933300000003 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.511046+0000", 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1179000000000002e-05 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.511478+0000", 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000431691 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.515262+0000", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037837470000000001 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.516939+0000", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001677092 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517121+0000", 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018196600000000001 2024-09-13T14:30:50.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517168+0000", 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7259999999999998e-05 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517211+0000", 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3179000000000001e-05 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241054 5.16 5:6ea6704d:::benchmark_data_smithi017_147622_object106425:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:20.715354+0000", 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 90.130545544, 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80656303200000001, 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241054 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715354+0000", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715354+0000", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715352+0000", 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715359+0000", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.494e-06 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715373+0000", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4795e-05 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.715380+0000", 2024-09-13T14:30:50.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1260000000000003e-06 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.717153+0000", 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001773681 2024-09-13T14:30:50.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.717161+0000", 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4370000000000001e-06 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.513622+0000", 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79646085899999997 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.513713+0000", 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1254000000000004e-05 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:30:50.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.513930+0000", 2024-09-13T14:30:50.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021684699999999999 2024-09-13T14:30:50.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:50.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.518850+0000", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0049200490000000001 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521782+0000", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029325330000000002 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521836+0000", 2024-09-13T14:30:50.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3436999999999997e-05 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521886+0000", 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0049000000000001e-05 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521917+0000", 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1081000000000003e-05 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241066 5.16 5:6e62d9cb:::benchmark_data_smithi017_147622_object106437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:20.728069+0000", 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 90.11783054, 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79693475899999999, 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241066 2024-09-13T14:30:50.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728069+0000", 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728069+0000", 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728068+0000", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728073+0000", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2530000000000004e-06 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728082+0000", 2024-09-13T14:30:50.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.721e-06 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728086+0000", 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9829999999999998e-06 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728127+0000", 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1417e-05 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:30:50.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728136+0000", 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3939999999999996e-06 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.515298+0000", 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.78716225799999995 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.515364+0000", 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6570999999999997e-05 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.515632+0000", 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00026756499999999999 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.518857+0000", 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032253949999999998 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521949+0000", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030911319999999999 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.524916+0000", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029669549999999999 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.524964+0000", 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8143999999999998e-05 2024-09-13T14:30:50.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525004+0000", 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9904000000000003e-05 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241067 5.16 5:6c2f9786:::benchmark_data_smithi017_147622_object106438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:20.728168+0000", 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 90.117731168999995, 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79700580499999996, 2024-09-13T14:30:50.966 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241067 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728168+0000", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728168+0000", 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728168+0000", 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728170+0000", 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3020000000000002e-06 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728176+0000", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6910000000000002e-06 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728179+0000", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.717e-06 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728215+0000", 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6888000000000003e-05 2024-09-13T14:30:50.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.728224+0000", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3869999999999995e-06 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517252+0000", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.78902817199999997 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517319+0000", 2024-09-13T14:30:50.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7520000000000004e-05 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.517550+0000", 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00023075400000000001 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521793+0000", 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042431730000000003 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.524635+0000", 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002841466 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525054+0000", 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041866000000000001 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525134+0000", 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0130999999999996e-05 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.972 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525174+0000", 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0326e-05 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241074 5.16 5:6bcbbcc1:::benchmark_data_smithi017_147622_object106445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:20.740530+0000", 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 90.105369775, 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.784772264, 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.973 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241074 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740530+0000", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740530+0000", 2024-09-13T14:30:50.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740528+0000", 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740534+0000", 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2739999999999998e-06 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740547+0000", 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3154e-05 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740552+0000", 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7729999999999999e-06 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740588+0000", 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6300999999999997e-05 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:20.740595+0000", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7109999999999998e-06 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.518882+0000", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.77828729299999999 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.518993+0000", 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011092299999999999 2024-09-13T14:30:50.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.519293+0000", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00030027500000000002 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.521929+0000", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026361890000000002 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.524759+0000", 2024-09-13T14:30:50.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028294370000000002 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525216+0000", 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00045754999999999998 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525259+0000", 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2500000000000003e-05 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.979 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525302+0000", 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3040999999999997e-05 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:25746 6.a 6:56d437ca:::benchmark_data_smithi017_185083_object25745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:30:46.240300+0000", 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4.6055990040000001, 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087525172999999998, 2024-09-13T14:30:50.980 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 25746 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240300+0000", 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240300+0000", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240299+0000", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240364+0000", 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4639000000000002e-05 2024-09-13T14:30:50.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240378+0000", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4827999999999999e-05 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240385+0000", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1770000000000003e-06 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.240629+0000", 2024-09-13T14:30:50.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000244319 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.241810+0000", 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011806360000000001 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.242164+0000", 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035486600000000001 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.245587+0000", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034229260000000002 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.245634+0000", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6344999999999998e-05 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.327735+0000", 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082101814999999995 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.327783+0000", 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7194000000000002e-05 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:30:46.327826+0000", 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2855999999999998e-05 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:30:50.986 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:30:51.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:51.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:51.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:51.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:51.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:51.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:51.462 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:51 smithi189 ceph-mon[90992]: pgmap v478: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:30:51.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:51.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:51.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:51.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 36772 36756 153.121 120.688 0.0050148 0.00652545 2024-09-13T14:30:51.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:51 smithi017 ceph-mon[107556]: pgmap v478: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:30:51.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:51 smithi017 ceph-mon[103001]: pgmap v478: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-09-13T14:30:51.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:51.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:30:51.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:51.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:51.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:51.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:51.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:51.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:52.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:52.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:52.154 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:52.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:52.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:52.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:52.421 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:52.433 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:30:52.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:30:52.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:52.475 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:30:52.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:30:52.517 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:30:52.518 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:30:52.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:52.557 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:30:52.558 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:30:52.586 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:52.593 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:30:52.594 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:30:52.612 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2593436157' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:52.627 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:30:52.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2593436157' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:52.627 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:30:52.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2593436157' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:30:52.661 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:30:52.661 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:30:52.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 38674 38658 150.98 118.875 0.00550788 0.00661837 2024-09-13T14:30:52.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:52.694 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:30:52.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:30:52.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:52.819 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:52.848 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:52.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:52.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:52.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:52.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:53.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:53.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:53.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:53.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:53.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:53.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:53.475 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-13T14:30:53.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:53.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:53.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:53.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:53 smithi189 ceph-mon[90992]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.43k op/s 2024-09-13T14:30:53.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:30:53.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:53.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 40443 40427 148.602 110.562 0.00629381 0.00672472 2024-09-13T14:30:53.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:53.801 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:53.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:53.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:53 smithi017 ceph-mon[103001]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.43k op/s 2024-09-13T14:30:53.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:30:53.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:53 smithi017 ceph-mon[107556]: pgmap v479: 99 pgs: 99 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.43k op/s 2024-09-13T14:30:53.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:30:53.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:53.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:54.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:54.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:54.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:54.158 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:54.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:54.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:54.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:54.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:54.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:54.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:54.563 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:54.563 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:54.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:54.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:54.674 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 42115 42099 146.152 104.5 0.00546254 0.00683574 2024-09-13T14:30:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:30:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:54 smithi017 ceph-mon[103001]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:30:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:30:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:54 smithi017 ceph-mon[107556]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:30:54.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2967982820' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:30:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:54 smithi189 ceph-mon[90992]: osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:30:54.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:54.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:54.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:54.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:55.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:55.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:55.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:55.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:55.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:55.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:30:55.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:55.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:55.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:55.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:55.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:55.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:55.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:55.625 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:55.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 43719 43703 143.735 100.25 0.0151946 0.00695161 2024-09-13T14:30:55.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:55 smithi189 ceph-mon[90992]: pgmap v481: 99 pgs: 99 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:30:55.758 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:55 smithi017 ceph-mon[103001]: pgmap v481: 99 pgs: 99 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:30:55.758 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:55 smithi017 ceph-mon[107556]: pgmap v481: 99 pgs: 99 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:30:55.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:55.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:55.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:55.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:55.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:56.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:56.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:56.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:56.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:56.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:56.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:56.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:56.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:56.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:56.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:56.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:56.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:56.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:56.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:30:56.674888+0000 min lat: 0.00121076 max lat: 0.0959274 avg lat: 0.0070532 2024-09-13T14:30:56.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:30:56.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 15 45360 45345 141.68 102.625 0.014693 0.0070532 2024-09-13T14:30:56.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:56.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:56.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:56.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:56.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:56.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:57.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:57.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:57.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:57.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:57.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:57.344 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:57.446 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:57.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:57.507 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:30:57.508 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:30:57.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:57.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[103001]: pgmap v482: 99 pgs: 99 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:30:57] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[107556]: pgmap v482: 99 pgs: 99 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:57.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:57.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:57.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 47204 47188 140.418 115.188 0.00752016 0.00711651 2024-09-13T14:30:57.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:57 smithi189 ceph-mon[90992]: pgmap v482: 99 pgs: 99 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:30:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:30:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:57.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:57.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:57.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:57.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:58.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:30:58.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:30:58.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:58.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:58.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:30:58.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:58.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:58.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:58.462 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:58.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:58.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:30:58.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:30:58.623 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:30:58.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 48940 48924 138.967 108.5 0.00464491 0.00719106 2024-09-13T14:30:58.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:58.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:58.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:58.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:30:58.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:58.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:30:58.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:30:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:59.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:30:59.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:30:59.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:59.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:30:59.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:59.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:59.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:30:59.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:30:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:30:59.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:30:59.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:59.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:59.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 50539 50523 137.269 99.9375 0.0206083 0.00727961 2024-09-13T14:30:59.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:30:59.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:59.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:30:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:30:59 smithi189 ceph-mon[90992]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-13T14:30:59.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:59.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:30:59.902 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:30:59 smithi017 ceph-mon[103001]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-13T14:30:59.903 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:30:59 smithi017 ceph-mon[107556]: pgmap v483: 99 pgs: 99 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-13T14:30:59.941 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:30:59.973 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:30:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:30:59.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:00.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:00.122 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:00.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:00.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:00.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:00.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:00.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:00.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:00.488 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:00.504 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:31:00.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:00.528 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:00.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:00.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:00.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:00.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:00.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 51957 51941 135.242 88.625 0.00692311 0.00738926 2024-09-13T14:31:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:00.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:00.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:00.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:00.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:00.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:01.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:01.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:01.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:01.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:01.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:01.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:01.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:01.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:01.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:01.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:01 smithi189 ceph-mon[90992]: pgmap v484: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:31:01.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:01 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:01.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:01.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:01 smithi189 ceph-mon[90992]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:31:01.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:01.659 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:31:01.660 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:31:01.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 53541 53525 133.792 99 0.00809184 0.00746994 2024-09-13T14:31:01.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:01.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:01.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:01.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[103001]: pgmap v484: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:31:01.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:01.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:01.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[103001]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:31:01.897 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[107556]: pgmap v484: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:31:01.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:01.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3928553219' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:01.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:01 smithi017 ceph-mon[107556]: osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:31:01.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:01.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:02.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:02.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:02.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:02.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:02.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:02.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:02.350 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:31:02.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:31:02.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:31:02.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:02.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:02.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.619 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:02.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:02.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 55035 55019 132.238 93.375 0.00734756 0.00755644 2024-09-13T14:31:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:02.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:02.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:02.810 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:31:02.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:02.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:02.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:03.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:03.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:03.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:03.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:03.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:03.290 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:03.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:03.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:03.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:03.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:03.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:03.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 56485 56469 130.696 90.625 0.0213174 0.00764631 2024-09-13T14:31:03.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:03.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:03.753 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:03 smithi017 ceph-mon[103001]: pgmap v486: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:31:03.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:03 smithi017 ceph-mon[107556]: pgmap v486: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:31:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:03 smithi189 ceph-mon[90992]: pgmap v486: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:31:03.805 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:03.807 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:31:03.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:31:03.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:31:03.831 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:31:03.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:03.868 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:31:03.868 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:31:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:03.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:03.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:03.909 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:31:03.909 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:31:03.945 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:31:03.946 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:31:03.979 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:31:03.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:31:04.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:04.013 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:31:04.013 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:31:04.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:04.050 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:31:04.050 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:31:04.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:04.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:04.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:04.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:04.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:04.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:04.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:04.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:04.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:04.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 57903 57887 129.193 88.625 0.0104881 0.00773499 2024-09-13T14:31:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1213360903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:04 smithi017 ceph-mon[107556]: pgmap v487: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:31:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1213360903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:04 smithi017 ceph-mon[103001]: pgmap v487: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:31:04.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1213360903' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:04 smithi189 ceph-mon[90992]: pgmap v487: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:31:04.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:04.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:04.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:04.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:05.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:05.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:05.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:05.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:05.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:31:05.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:05.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:05.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:05.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:05.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35349.0:59206 6.9 e180/173 6:958c0910:::benchmark_data_smithi017_185083_object59205:head v 180'3694, mlcod=180'3692)", 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:31:05.565348+0000", 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0052387329999999998, 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0052642909999999999, 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:31:05.596 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565348+0000", 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565348+0000", 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565347+0000", 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565388+0000", 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0884999999999999e-05 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:31:05.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565402+0000", 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.383e-05 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565407+0000", 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2630000000000003e-06 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:31:05.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565466+0000", 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8785e-05 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.565487+0000", 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0632000000000002e-05 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.600 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:59209 6.8 6:15d3a359:::benchmark_data_smithi017_185083_object59208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:31:05.566131+0000", 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0044556489999999999, 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045461149999999999, 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 59209 2024-09-13T14:31:05.601 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566131+0000", 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566131+0000", 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566130+0000", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566163+0000", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3473000000000003e-05 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566176+0000", 2024-09-13T14:31:05.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2568999999999999e-05 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566182+0000", 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6609999999999997e-06 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566263+0000", 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1615999999999999e-05 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566384+0000", 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000120732 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566511+0000", 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012714200000000001 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.570616+0000", 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041050469999999997 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:31:05.606 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:59210 6.8 6:1153e993:::benchmark_data_smithi017_185083_object59209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:31:05.566194+0000", 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0043925379999999997, 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045586749999999999, 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 59210 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.607 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566194+0000", 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566194+0000", 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:31:05.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566194+0000", 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566219+0000", 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5258000000000001e-05 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566226+0000", 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.489e-06 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.566230+0000", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8739999999999997e-06 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.568648+0000", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002418103 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.568772+0000", 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012457599999999999 2024-09-13T14:31:05.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.568911+0000", 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013888299999999999 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:31:05.611 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35349.0:59200 6.6 e180/173 6:6a6f57eb:::benchmark_data_smithi017_185083_object59199:head v 180'3695, mlcod=180'3691)", 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:31:05.567332+0000", 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.003254954, 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034840700000000001, 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567332+0000", 2024-09-13T14:31:05.612 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567332+0000", 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567331+0000", 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:31:05.613 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567447+0000", 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000115217 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567456+0000", 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6080000000000002e-06 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.567461+0000", 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0010000000000001e-06 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.614 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569845+0000", 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023843699999999998 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569866+0000", 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0407999999999998e-05 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.615 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.616 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.616 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35349.0:59211 6.9 e180/173 6:9478f524:::benchmark_data_smithi017_185083_object59210:head v 180'3695, mlcod=180'3692)", 2024-09-13T14:31:05.616 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:31:05.569786+0000", 2024-09-13T14:31:05.616 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.00080115600000000003, 2024-09-13T14:31:05.616 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00109366, 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569786+0000", 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.617 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569786+0000", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569785+0000", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569915+0000", 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012961099999999999 2024-09-13T14:31:05.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569924+0000", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8220000000000002e-06 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569929+0000", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8980000000000004e-06 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569964+0000", 2024-09-13T14:31:05.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5144000000000001e-05 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:31:05.569994+0000", 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9184999999999999e-05 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 5 2024-09-13T14:31:05.620 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:31:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:05.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:05.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 59382 59366 127.926 92.4375 0.00477811 0.00781205 2024-09-13T14:31:05.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:05.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:05.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:05.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:05.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:05.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:05.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:06.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:06.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:06.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:06.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:06.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:06.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:06.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:06.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:06.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:06.621 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:31:06.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:06.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 60826 60810 126.67 90.25 0.017345 0.00788932 2024-09-13T14:31:06.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:06.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:06.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:06.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:06.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:06.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:07.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:07.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:07.129 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:31:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:06 smithi017 ceph-mon[103001]: pgmap v488: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:31:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:06 smithi017 ceph-mon[107556]: pgmap v488: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:31:07.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:07.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:06 smithi189 ceph-mon[90992]: pgmap v488: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:31:07.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:07.300 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:07.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:31:07.358 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:07.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:07.489 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:07.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:07.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:07.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:07.675 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 62159 62143 125.271 83.3125 0.0100678 0.00797657 2024-09-13T14:31:07.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:07] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-13T14:31:07.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:07.807 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:07.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:07.913 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:31:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:07.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:07.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:31:08.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:08.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:08.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:31:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:31:08.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:08.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:08.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:08.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:08.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:08.539 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:08.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:08.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:08.676 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 63575 63559 124.121 88.5 0.00714335 0.0080521 2024-09-13T14:31:08.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:08.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:08.917 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:08.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:08.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:09.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:09.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:09.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[103001]: pgmap v489: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[103001]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[107556]: pgmap v489: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:31:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:08 smithi017 ceph-mon[107556]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:31:09.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:09.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:08 smithi189 ceph-mon[90992]: pgmap v489: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-09-13T14:31:09.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3730573444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:31:09.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:08 smithi189 ceph-mon[90992]: osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:31:09.342 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:09.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:09.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:09.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:09.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:09.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:31:09.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:09.676 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 64948 64932 122.96 85.8125 0.0102594 0.0081277 2024-09-13T14:31:09.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:09.788 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:09.818 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:09.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:09.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:09.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:10.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:10.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:10.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:10.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:10.229 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:10.324 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:10.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:10.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:10.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:10.467 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:10.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:10.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:10.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:10.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:10.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 66101 66085 121.461 72.0625 0.0186871 0.00822754 2024-09-13T14:31:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:10.731 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:31:10.732 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:31:10.732 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:31:10.732 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:31:10.732 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:31:10.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:10.813 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:10.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:10.925 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:31:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[103001]: pgmap v491: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:31:10.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3199258835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[107556]: pgmap v491: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:31:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3199258835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:10.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:10.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:10 smithi189 ceph-mon[90992]: pgmap v491: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:31:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3199258835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:11.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:11.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:11.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:11.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:11.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:11.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:11.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:11.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:11.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:11.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:11.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:11.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:11.676 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 67433 67417 120.369 83.25 0.00666064 0.00830279 2024-09-13T14:31:11.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:11.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:11.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:11.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:11.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:11.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:12.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:12.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:12.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[103001]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:11 smithi017 ceph-mon[107556]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:31:12.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:11 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:11 smithi189 ceph-mon[90992]: osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:31:12.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:12.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:12.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:12.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:12.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:12.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:31:12.578 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:12.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:12.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 68620 68604 119.086 74.1875 0.00987841 0.00839242 2024-09-13T14:31:12.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:12.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:12.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:12.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:12.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:13.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:13.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:13.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:13.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:13.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:13.311 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:31:13.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:13.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[107556]: pgmap v493: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:31:13.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:13.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:13.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[103001]: pgmap v493: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:31:13.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:13.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:13.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:13.449 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:13 smithi189 ceph-mon[90992]: pgmap v493: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:31:13.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:13.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:13.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:13.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:13.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:13.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 70040 70024 118.266 88.75 0.00962851 0.00845058 2024-09-13T14:31:13.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:13.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:13.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:13.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:13.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:13.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:14.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:14.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:14.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:14.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:14.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:14.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:14.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:14.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:14.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:14.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:14.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 71439 71423 117.455 87.4375 0.0101723 0.00850924 2024-09-13T14:31:14.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:14.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:14.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:14.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:14.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:14.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:15.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:15.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:15.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:15.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:15.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:15.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:15.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:15.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:15 smithi017 ceph-mon[103001]: pgmap v494: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:31:15.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:15 smithi017 ceph-mon[107556]: pgmap v494: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:31:15.497 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:15 smithi189 ceph-mon[90992]: pgmap v494: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:31:15.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:15.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:15.627 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:31:15.628 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:31:15.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:15.665 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:31:15.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:31:15.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 15 72597 72582 116.3 72.4375 0.015408 0.00859301 2024-09-13T14:31:15.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:15.703 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:31:15.703 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:31:15.737 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:31:15.738 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:31:15.767 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:31:15.767 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:31:15.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:15.799 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:31:15.799 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:31:15.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:15.833 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:31:15.833 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:31:15.865 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:31:15.865 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:31:15.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:31:15.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:15.972 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:31:15.973 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:31:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:15.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:15.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:16.077 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:16.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:16.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:16.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:16.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:16.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:16.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:16.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1015717078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1015717078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1015717078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:16.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:16.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:16.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:31:16.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:16.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:31:16.677423+0000 min lat: 0.00121076 max lat: 0.122407 avg lat: 0.00867699 2024-09-13T14:31:16.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:16.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 73735 73719 115.169 71.0625 0.0101458 0.00867699 2024-09-13T14:31:16.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:16.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:16.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:16.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:16.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:16.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:16.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:17.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:17.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:17.130 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:31:17.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:17.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:17.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:17.392 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:17 smithi189 ceph-mon[90992]: pgmap v495: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:31:17.393 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/463512908' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.393 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:17 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:17.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[103001]: pgmap v495: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/463512908' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:17] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.51.0" 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[107556]: pgmap v495: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/463512908' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:17 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:31:17.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:17.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:17.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:17.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:17.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 74946 74930 114.206 75.6875 0.00408126 0.00875112 2024-09-13T14:31:17.777 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:17.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:17.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:17.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:17.941 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:17.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:17.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:18.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:18.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:18.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:18.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:18.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:18 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:31:18.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:18 smithi189 ceph-mon[90992]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:31:18.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:18.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:18 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:31:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:18 smithi017 ceph-mon[103001]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:31:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:18 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:31:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:18 smithi017 ceph-mon[107556]: osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:31:18.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:18.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:18.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:18.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:18.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:18.677 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 76218 76202 113.38 79.5 0.0104602 0.00881528 2024-09-13T14:31:18.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:18.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:18.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:18.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:18.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:18.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:19.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:19.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:19.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:19.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:19.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:19.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:19.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:19.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:19.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:19.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:19.678 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 77309 77293 112.327 68.1875 0.0119637 0.00889679 2024-09-13T14:31:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[103001]: pgmap v497: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:31:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[107556]: pgmap v497: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:31:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:19 smithi189 ceph-mon[90992]: pgmap v497: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:31:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:19.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:19.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:31:19.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:19.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:31:19.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:19.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:19.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:20.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:20.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:20.235 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:20.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:20.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:20.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:20.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:20.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:20.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:20.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:20.679 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 78423 78407 111.355 69.625 0.0126086 0.00897549 2024-09-13T14:31:20.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:20.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:20.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:20.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:20.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:20.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:20.980 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:20.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:21.081 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:21.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:21.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:21.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:21.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:21.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:21.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:21.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:21.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:21.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:21.679 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 79702 79686 110.657 79.9375 0.0074153 0.00903096 2024-09-13T14:31:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:21 smithi017 ceph-mon[103001]: pgmap v498: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:31:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:21 smithi017 ceph-mon[107556]: pgmap v498: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:31:21.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:21 smithi189 ceph-mon[90992]: pgmap v498: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:31:21.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:21.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:21.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:21.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:22.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:22.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:22.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:22.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:22.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:22.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:22.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:22.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:22.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:22.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:22.679 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 80857 80841 109.82 72.1875 0.0118936 0.00910053 2024-09-13T14:31:22.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:22.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:22.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:22.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:22.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:23.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:23.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:23.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:23.164 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:23.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:23.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:23.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:23.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:23.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:23.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:23.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:23.679 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 82085 82069 109.117 76.75 0.00486383 0.00915871 2024-09-13T14:31:23.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:23 smithi017 ceph-mon[103001]: pgmap v499: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:31:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:23 smithi017 ceph-mon[107556]: pgmap v499: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:31:23.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:23 smithi189 ceph-mon[90992]: pgmap v499: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:31:23.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:23.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:23.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:23.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:23.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:24.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:24.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:24.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:24.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:24.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:24.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:24.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:24.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:24.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:24.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 83179 83163 108.264 68.375 0.022883 0.00923213 2024-09-13T14:31:24.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:24.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:24.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:24.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:24.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:25.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:25.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:25.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:25.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:25.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:25.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:25.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:25.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:25.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:25.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:25.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 84197 84181 107.353 63.625 0.0169362 0.00930963 2024-09-13T14:31:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:25 smithi017 ceph-mon[107556]: pgmap v500: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:31:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:25 smithi017 ceph-mon[103001]: pgmap v500: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:31:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:25 smithi189 ceph-mon[90992]: pgmap v500: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:31:25.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:25.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:25.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:25.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:25.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:26.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:26.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:26.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:26.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:26.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:26.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:26.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:26.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:26.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:26.643 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:26.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 85267 85251 106.544 66.875 0.0120979 0.00938097 2024-09-13T14:31:26.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:26.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:26.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:26.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:27.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:27.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[103001]: pgmap v501: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:27.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[107556]: pgmap v501: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:27.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:26 smithi189 ceph-mon[90992]: pgmap v501: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:27.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:27.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:27.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:27.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:27.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:27.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:27.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:27.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:27.680 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:27] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:31:27.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 86363 86347 105.798 68.5 0.0143751 0.00944665 2024-09-13T14:31:27.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:27.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:27.897 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:27.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:27.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:27.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:27.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:28.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:28.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:28.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:28.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:28.229 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:31:28.229 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:31:28.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:28.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:28.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:31:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:28.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:28.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:28.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:28.494 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:28.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:28.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:28.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:28.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 87388 87372 104.995 64.0625 0.00720118 0.00951913 2024-09-13T14:31:28.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:28.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:28.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:28.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:28.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:29.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:29.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:29.110 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:31:29.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:29.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:29.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:29.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:29.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[103001]: pgmap v502: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:31:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3102330049' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[107556]: pgmap v502: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:31:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3102330049' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.466 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:29 smithi189 ceph-mon[90992]: pgmap v502: 99 pgs: 99 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:31:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3102330049' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:31:29.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:29.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:29.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 88579 88563 104.418 74.4375 0.0153494 0.00957191 2024-09-13T14:31:29.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:29.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:29.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:29.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:29.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:29.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:30.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:30.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:30.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:30.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:30.266 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:30.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:30.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:30.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[103001]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:30 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:30 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:30 smithi189 ceph-mon[90992]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:31:30.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:30 smithi017 ceph-mon[107556]: osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:31:30.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:30.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:30.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:30.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:30.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:30.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 89751 89735 103.841 73.25 0.0123879 0.00962518 2024-09-13T14:31:30.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:30.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:30.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:30.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:30.946 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:30.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:30.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:31.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:31.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:31.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:31.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:31.304 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:31 smithi017 ceph-mon[103001]: pgmap v504: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:31:31.305 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:31 smithi017 ceph-mon[107556]: pgmap v504: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:31:31.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:31.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:31.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:31 smithi189 ceph-mon[90992]: pgmap v504: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:31:31.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:31.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:31.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:31.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 90889 90873 103.246 71.125 0.0087438 0.00968099 2024-09-13T14:31:31.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:31.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:31:31.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:31.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:31.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:31.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:32.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:32.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:32.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:32.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:32.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:32.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:32.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:32.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:32.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:32.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:32.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 91887 91871 102.516 62.375 0.0113511 0.00974948 2024-09-13T14:31:32.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:32.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:31:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:32.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:32.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:32.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:32.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:32.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:33.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:33.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:33.154 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:31:33.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.198 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:33.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:33.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:33.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:33.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[107556]: pgmap v505: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/649099783' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[103001]: pgmap v505: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/649099783' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:33.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:33 smithi189 ceph-mon[90992]: pgmap v505: 99 pgs: 99 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:31:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/649099783' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:31:33.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:33.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:33.681 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 92885 92869 101.812 62.375 0.0114279 0.00981685 2024-09-13T14:31:33.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:33.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:33.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:33.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:33.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:33.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:33.964 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:31:33.964 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:31:33.983 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:34.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:31:34.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:31:34.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:34.041 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:31:34.041 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:31:34.106 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:31:34.106 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:31:34.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:34.136 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:31:34.137 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:31:34.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:34.170 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:31:34.170 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:31:34.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:31:34.205 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:31:34.235 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:31:34.235 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:31:34.253 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:34.280 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:34.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:34.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:34.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:31:34.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[107556]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:31:34.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2675801386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:34.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:31:34.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[103001]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:31:34.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2675801386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:34.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:31:34.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:34 smithi189 ceph-mon[90992]: osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:31:34.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2675801386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:34.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:34.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:34.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:34.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 93935 93919 101.188 65.625 0.00335487 0.00987173 2024-09-13T14:31:34.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:34.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:34.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:34.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:34.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:34.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:34.989 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:35.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:35.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:35.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:35.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:35.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:35 smithi017 ceph-mon[107556]: pgmap v507: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:31:35.284 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:35 smithi017 ceph-mon[103001]: pgmap v507: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:31:35.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:35.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:35.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:35 smithi189 ceph-mon[90992]: pgmap v507: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T14:31:35.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:35.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:35.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:35.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 94968 94952 100.567 64.5625 0.0210133 0.0099386 2024-09-13T14:31:35.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:35.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:35.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:35.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:35.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:36.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:36.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:36.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:36.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:36.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:36.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:36.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:36.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:36.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:36.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:36.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:31:36.682155+0000 min lat: 0.00121076 max lat: 0.145391 avg lat: 0.0099921 2024-09-13T14:31:36.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:36.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 96061 96045 100.029 68.3125 0.0139672 0.0099921 2024-09-13T14:31:36.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:36.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:36.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:36.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:31:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:36.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:36.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:37.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:37.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:37.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:37.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:37 smithi017 ceph-mon[103001]: pgmap v508: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:31:37.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:37.299 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:37.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:37.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:37.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:37] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:31:37.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:37 smithi017 ceph-mon[107556]: pgmap v508: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:31:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:37 smithi189 ceph-mon[90992]: pgmap v508: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:31:37.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:37.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:37.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:37.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:37.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:37.682 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 15 97094 97079 99.4488 64.625 0.0181558 0.0100503 2024-09-13T14:31:37.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:37.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:37.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:37.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:37.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:38.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:38.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:38.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:31:38.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:38.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:38.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:38.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:38.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:38.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:38.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:38.683 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 98118 98102 98.8742 63.9375 0.0133636 0.010109 2024-09-13T14:31:38.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:38.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:38.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:38.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:38.873 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:31:38.873 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:31:38.873 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:31:38.873 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:31:38.873 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:31:38.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:38.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:38.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:39.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:39.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:39.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:39.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:39.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:39.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:39 smithi017 ceph-mon[103001]: pgmap v509: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-13T14:31:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:39 smithi017 ceph-mon[107556]: pgmap v509: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-13T14:31:39.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:39 smithi189 ceph-mon[90992]: pgmap v509: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2024-09-13T14:31:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:39.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:39.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:39.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:39.683 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 99189 99173 98.3672 66.9375 0.00920567 0.0101608 2024-09-13T14:31:39.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:39.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:39.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:39.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:39.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:40.008 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:40.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:40.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:40.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:40.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:40.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:40.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:40.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:40.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:31:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:40.538 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:40.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:40.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:40.683 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 100213 100197 97.8298 64 0.00668993 0.0102167 2024-09-13T14:31:40.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:40.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:40.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:40.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:40.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:40.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:41.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:41.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:41.122 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:41.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:41.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:41.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:41.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:41.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:41 smithi017 ceph-mon[103001]: pgmap v510: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:41 smithi017 ceph-mon[107556]: pgmap v510: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:41.500 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:41 smithi189 ceph-mon[90992]: pgmap v510: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:41.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:41.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:41.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 101243 101227 97.3137 64.375 0.0170524 0.010271 2024-09-13T14:31:41.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:41.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:41.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:41.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:41.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:41.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:42.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:42.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:42.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:42.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:42.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:42.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:42.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:42.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:42.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.460 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:42.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:42.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:42.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:42.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 102199 102183 96.7446 59.75 0.013334 0.0103305 2024-09-13T14:31:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:42.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:42.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:42.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:42.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:42.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:42.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:43.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:43.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:43.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:43.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:43.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:43.236 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:31:43.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:43.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:43.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[107556]: pgmap v511: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:43.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1867224848' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[103001]: pgmap v511: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:43.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1867224848' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:43 smithi189 ceph-mon[90992]: pgmap v511: 99 pgs: 99 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:31:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1867224848' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:31:43.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:43.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:43.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:43.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:43.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:43.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 103201 103185 96.2353 62.625 0.0258596 0.0103856 2024-09-13T14:31:43.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:43.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:43.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:43.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:43.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:44.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:44.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:44.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:44.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:44.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:44.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:44.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:44.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:31:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:31:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[103001]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:31:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:31:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:31:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:44 smithi017 ceph-mon[107556]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:31:44.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:44 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:31:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:44 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:31:44.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:31:44.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:44 smithi189 ceph-mon[90992]: osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:31:44.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:44.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:44.686 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 104253 104237 95.7869 65.75 0.0128379 0.0104345 2024-09-13T14:31:44.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:44.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:44.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:44.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:44.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:44.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:44.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:45.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:45.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:45.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:45.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:45.174 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:45.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:45.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:45.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:45.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:45 smithi017 ceph-mon[103001]: pgmap v513: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:45 smithi017 ceph-mon[107556]: pgmap v513: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:45 smithi189 ceph-mon[90992]: pgmap v513: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:45.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:45.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:45.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 105265 105249 95.3154 63.25 0.00641797 0.0104863 2024-09-13T14:31:45.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:45.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:45.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:45.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:45.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:45.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:46.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:46.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:46.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:46.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:31:46.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:46.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:46.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:46.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:46.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:46.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:46.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 106168 106152 94.7599 56.4375 0.0197662 0.0105475 2024-09-13T14:31:46.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:46.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:46.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:46.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:46.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:46.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:46.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:47.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:47.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:47.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:47.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:47.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:47.262 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:31:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:47.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:47.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:47.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:47.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:47.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:47.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:47.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:47.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:47.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:47.685 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 107175 107159 94.3117 62.9375 0.0209711 0.010598 2024-09-13T14:31:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[107556]: pgmap v514: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:31:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/177572504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:47] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:31:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[103001]: pgmap v514: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:31:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/177572504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:47 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:47 smithi189 ceph-mon[90992]: pgmap v514: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:31:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/177572504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:47 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:31:47.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:47.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:47.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:47.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:47.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:47.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:48.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:48.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:48.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:48.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:48.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:48.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:48.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:31:48.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:48 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:31:48.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:48 smithi017 ceph-mon[103001]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:31:48.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:48 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:31:48.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:48 smithi017 ceph-mon[107556]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:31:48.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:48.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:48.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:48.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:48.687 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 108148 108132 93.8442 60.8125 0.0162876 0.0106506 2024-09-13T14:31:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:48 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:31:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:48 smithi189 ceph-mon[90992]: osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:31:48.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:48.785 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:31:48.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:31:48.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:48.829 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:31:48.829 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:31:48.869 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:31:48.870 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:31:48.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:48.918 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:31:48.918 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:31:48.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:48.920 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:48.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:48.951 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:31:48.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:31:48.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:31:48.983 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:31:49.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:31:49.013 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:31:49.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:49.049 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:31:49.049 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:31:49.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:49.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:49.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:49.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:49.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:49.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:49.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:49 smithi017 ceph-mon[103001]: pgmap v516: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-13T14:31:49.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/189403488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:49.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:49 smithi017 ceph-mon[107556]: pgmap v516: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-13T14:31:49.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/189403488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:49.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:49.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:49.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:49.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:49.687 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 109048 109032 93.3292 56.25 0.00515586 0.0107095 2024-09-13T14:31:49.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:49 smithi189 ceph-mon[90992]: pgmap v516: 99 pgs: 99 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-13T14:31:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/189403488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:31:49.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:49.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:49.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:49.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:49.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:49.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:50.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:50.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:50.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:50.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:50.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:50.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:31:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:31:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:50.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:50.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:50.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:50.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:50.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 110073 110057 92.9331 64.0625 0.0276708 0.0107546 2024-09-13T14:31:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:50.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:50.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:50.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:50.856 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:50.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:50.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:50.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:51.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:51.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:51.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:51.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:51.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:51.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:51 smithi017 ceph-mon[103001]: pgmap v517: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:51.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:51 smithi017 ceph-mon[107556]: pgmap v517: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:51.494 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:51.498 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:51.516 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:51.551 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:31:51.551 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:31:51.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:51.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:51.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 111000 110984 92.4665 57.9375 0.0101888 0.0108094 2024-09-13T14:31:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:51 smithi189 ceph-mon[90992]: pgmap v517: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:31:51.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:51.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:51.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:51.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:51.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:52.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:52.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:52.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:52.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:52.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:52.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:52.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:52.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:52.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:52.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:31:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:52.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:52.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 112019 112003 92.0878 63.6875 0.013256 0.0108535 2024-09-13T14:31:52.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:52.759 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:52.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:52.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:52.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:52.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:53.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:53.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:53.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:53.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:53.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:53.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:53.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:53.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:53.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:31:53.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:53.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:53 smithi017 ceph-mon[103001]: pgmap v518: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2024-09-13T14:31:53.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:53 smithi017 ceph-mon[107556]: pgmap v518: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2024-09-13T14:31:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:53.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:53.680 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:53.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 112883 112867 91.5931 54 0.0158743 0.0109122 2024-09-13T14:31:53.704 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:53 smithi189 ceph-mon[90992]: pgmap v518: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s 2024-09-13T14:31:53.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:53.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:53.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:53.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:53.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:54.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:54.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:54.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:54.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:54.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:54.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:54.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:54.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:54.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:54.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:54.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 113665 113649 91.0455 48.875 0.0451915 0.0109776 2024-09-13T14:31:54.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:54.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:54.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:54.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:54.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:54.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:55.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:55.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:55.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:55.204 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:55.301 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:31:55.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:55.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:55.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:55.465 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:55.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:55.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 114490 114474 90.5457 51.5625 0.0211935 0.0110392 2024-09-13T14:31:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:55 smithi017 ceph-mon[103001]: pgmap v519: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:31:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:55 smithi017 ceph-mon[107556]: pgmap v519: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:31:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:55.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:55.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:55.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:55 smithi189 ceph-mon[90992]: pgmap v519: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:31:55.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:31:55.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:55.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:55.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:55.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:56.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:56.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:56.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:56.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:56.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:56.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:56.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[103001]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:31:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:56 smithi017 ceph-mon[107556]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:31:56.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:56.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:56.601 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:56.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:56.688 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:31:56.688778+0000 min lat: 0.00121076 max lat: 0.145391 avg lat: 0.0110852 2024-09-13T14:31:56.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:56.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 115462 115446 90.1728 60.75 0.0224142 0.0110852 2024-09-13T14:31:56.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:56 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:31:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1354998998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:31:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:56 smithi189 ceph-mon[90992]: osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:31:56.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:56.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:56.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:56.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:57.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:57.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:57.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:57.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:57.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:57.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:57.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:57.398 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:57.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:57.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:57.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:57.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:57.628 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:31:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:31:57] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-13T14:31:57.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:57 smithi017 ceph-mon[103001]: pgmap v521: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:31:57.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:57 smithi017 ceph-mon[107556]: pgmap v521: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:31:57.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:57.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 116426 116410 89.8034 60.25 0.0186323 0.0111301 2024-09-13T14:31:57.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:57 smithi189 ceph-mon[90992]: pgmap v521: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:31:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:57.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:31:57.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:31:57.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:57.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:57.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:58.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:58.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:58.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:31:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:31:58.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:58.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:58.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:58.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:58.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 117337 117321 89.4026 56.9375 0.0155856 0.0111792 2024-09-13T14:31:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:58.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:31:58.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:31:58.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:58.968 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:58.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:31:59.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:59.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:59.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:59.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:59.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:31:59.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:59.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:31:59.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:31:59.432 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:31:59.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:31:59.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:31:59.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:31:59.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:31:59.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 118241 118225 89.0061 56.5 0.0133718 0.0112299 2024-09-13T14:31:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:59 smithi017 ceph-mon[103001]: pgmap v522: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:31:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:31:59 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:31:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:59 smithi017 ceph-mon[107556]: pgmap v522: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:31:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:31:59 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:31:59.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:31:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:59 smithi189 ceph-mon[90992]: pgmap v522: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2024-09-13T14:31:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:31:59 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:31:59.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:31:59.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:31:59.915 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:31:59.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:31:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:31:59.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:00.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:00.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:00.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:00.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:00.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:00.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:00.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:32:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:00 smithi189 ceph-mon[90992]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:32:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:00.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:00.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:32:00.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:00 smithi017 ceph-mon[107556]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:32:00.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4147889283' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:32:00.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:00 smithi017 ceph-mon[103001]: osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:32:00.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:00.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:32:00.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:00.689 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 119199 119183 88.6586 59.875 0.0135032 0.0112737 2024-09-13T14:32:00.746 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:00.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:00.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:00.903 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:00.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:01.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:01.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:01.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:01.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:01.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:01.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:01.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:01.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:01 smithi189 ceph-mon[90992]: pgmap v524: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:32:01.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:01.471 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:01.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:01.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:01.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:01 smithi017 ceph-mon[103001]: pgmap v524: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:32:01.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:01 smithi017 ceph-mon[107556]: pgmap v524: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:32:01.691 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 120105 120089 88.2799 56.625 0.0075982 0.0113212 2024-09-13T14:32:01.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:01.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:01.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:01.936 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:01.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:02.031 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:32:02.031 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:32:02.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:02.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:02.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:02.220 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:02.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:02.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:02.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.482 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:02.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:02.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:02.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:02.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 121004 120988 87.9068 56.1875 0.0104801 0.011371 2024-09-13T14:32:02.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:02.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:02.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:02.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:02.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:03.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:03.027 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:03.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:03.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:03.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:03.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:03.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:03.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:03.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:03.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:03.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:03.618 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:32:03.618 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:32:03.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:03 smithi017 ceph-mon[103001]: pgmap v525: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2024-09-13T14:32:03.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:03 smithi017 ceph-mon[107556]: pgmap v525: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2024-09-13T14:32:03.660 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:32:03.660 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:32:03.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:03.691 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 121931 121915 87.5623 57.9375 0.00961541 0.0114153 2024-09-13T14:32:03.701 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:32:03.701 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:32:03.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:03.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:03.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:32:03.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:32:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:03 smithi189 ceph-mon[90992]: pgmap v525: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2024-09-13T14:32:03.767 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:32:03.767 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:32:03.797 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:32:03.798 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:32:03.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:03.828 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:32:03.828 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:32:03.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:03.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:32:03.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:32:03.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:03.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:03.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:04.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:04.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:32:04.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:04.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:04.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:04.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:04.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:04.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/498739571' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:04.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/498739571' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:04.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:04.561 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:04.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:04.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:04.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 122805 122789 87.188 54.625 0.0077011 0.0114643 2024-09-13T14:32:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:04.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/498739571' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:04.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:04.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:04.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:04.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:04.957 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:05.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:05.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:05.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:05.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:05.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:05.290 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:05.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:05.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:05 smithi017 ceph-mon[107556]: pgmap v526: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:32:05.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:05 smithi017 ceph-mon[103001]: pgmap v526: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:32:05.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:05.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:05.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:05.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 123661 123645 86.8095 53.5 0.00912541 0.0115145 2024-09-13T14:32:05.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:05 smithi189 ceph-mon[90992]: pgmap v526: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:32:05.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:05.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:05.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:05.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:05.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:06.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:06.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:06.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:06.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:06.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:06.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:06.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:06.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:06.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:06.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:06.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0108 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 124609 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 86.5237 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 36.2491 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 230.938 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 48.875 2024-09-13T14:32:06.692 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1384 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 579.986 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3695 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 782 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0115533 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.00813798 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.145391 2024-09-13T14:32:06.693 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00121076 2024-09-13T14:32:06.698 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:32:06.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:06.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:06.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:06.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:06.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:06.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:32:06.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:07.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:07.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:07.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:07.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:07.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:07.449 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:07.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:07.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:07.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:07.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:07 smithi017 ceph-mon[103001]: pgmap v527: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T14:32:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:07] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-13T14:32:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:07 smithi017 ceph-mon[107556]: pgmap v527: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T14:32:07.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:07.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:07.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:07 smithi189 ceph-mon[90992]: pgmap v527: 99 pgs: 99 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T14:32:07.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:07.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:07.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:07.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:08.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:08.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:08.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:08.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:08.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:08.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:08.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:08.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:08.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:08.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:08.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:08.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:08.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:09.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:09.129 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:09.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:09.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:09.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:09.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:09.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:09.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:09.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:09.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:09.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:09 smithi017 ceph-mon[103001]: pgmap v528: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 913 op/s 2024-09-13T14:32:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:09 smithi017 ceph-mon[107556]: pgmap v528: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 913 op/s 2024-09-13T14:32:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:09 smithi189 ceph-mon[90992]: pgmap v528: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 913 op/s 2024-09-13T14:32:09.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:09.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:09.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:09.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:09.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:10.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:10.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:10.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:10.214 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:32:09 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=sqlstore.transactions t=2024-09-13T14:32:09.913894845Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:32:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:10.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:10.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:10.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:10.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:10.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:10.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:10.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:10.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:10.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:10.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:10.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:10.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:10.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:10.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:11.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:11.058 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:11.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:11.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:11.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:11.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:11.388 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:32:11.423 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:11.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:11.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:11.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:11.697 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[103001]: pgmap v529: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 47 MiB/s wr, 861 op/s 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1582474372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[107556]: pgmap v529: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 47 MiB/s wr, 861 op/s 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1582474372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:11 smithi189 ceph-mon[90992]: pgmap v529: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 97 B/s rd, 47 MiB/s wr, 861 op/s 2024-09-13T14:32:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1582474372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:11.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:11.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:11.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:11.978 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:11.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:11.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:12.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:12.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:12.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:12.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:12.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:12.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:12.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:12.416 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:32:12.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:12.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:12.656 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:12.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:12.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:12.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:12.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:32:12.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2803591334' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2803591334' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:32:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2803591334' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:32:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:12.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:12.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:12.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:12.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:12.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:13.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:13.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:13.205 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:13.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:13.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:13.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:13.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:13.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:13.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:13.624 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:13.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[107556]: pgmap v531: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 902 op/s 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[107556]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[103001]: pgmap v531: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 902 op/s 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:32:13.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:13 smithi017 ceph-mon[103001]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:32:13.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:32:13.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:13 smithi189 ceph-mon[90992]: pgmap v531: 99 pgs: 99 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 902 op/s 2024-09-13T14:32:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:13 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:32:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:13 smithi189 ceph-mon[90992]: osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:32:13.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:13.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:13.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:13.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:13.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:14.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:14.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:14.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:14.096 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:32:14.096 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:32:14.097 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:32:14.097 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:32:14.097 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:32:14.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:14.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:14.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:14.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:14.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:14.468 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:14.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:14.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:14.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:32:14.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:14.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:14.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:32:14.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:14.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:14.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:15.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:15.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:15.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:15.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:15.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:15.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:15.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:15.377 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:15.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:15.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:15.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:15.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:15.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:15.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:15 smithi017 ceph-mon[103001]: pgmap v533: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 1.02k op/s 2024-09-13T14:32:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:15 smithi017 ceph-mon[107556]: pgmap v533: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 1.02k op/s 2024-09-13T14:32:15.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:15.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:15 smithi189 ceph-mon[90992]: pgmap v533: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 1.02k op/s 2024-09-13T14:32:15.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:15.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:15.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:15.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:15.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:15.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:16.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:16.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:16.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:16.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:16.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:16.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:16.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:16.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:16.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:16.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:16.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:16.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:16.859 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:16.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:16.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:16.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:17.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:17.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:17.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:17.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:17.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:17.364 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:17.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:17.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:17.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:17.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:17.539 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:17 smithi017 ceph-mon[103001]: pgmap v534: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 949 op/s 2024-09-13T14:32:17.539 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:17] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:32:17.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:17 smithi017 ceph-mon[107556]: pgmap v534: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 949 op/s 2024-09-13T14:32:17.594 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:32:17.594 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:32:17.594 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:32:17.594 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:32:17.603 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:32:17.603 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:32:17.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:17.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:17.653 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:32:17.654 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:32:17.692 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:32:17.692 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:32:17.730 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:32:17.730 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:32:17.746 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:17.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:17 smithi189 ceph-mon[90992]: pgmap v534: 99 pgs: 99 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 949 op/s 2024-09-13T14:32:17.763 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:32:17.763 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:32:17.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:17.797 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:32:17.797 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:32:17.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:17.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:17.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:17.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:18.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:18.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:18.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:18.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:18.286 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:18.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:18.421 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:18.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:18.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.578 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:18.679 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1512408429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1512408429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:18.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1512408429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:18.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:18.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:18.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:18.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:18.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:19.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:19.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:19.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:19.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:19.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:19.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:19.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:19.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:19.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:19.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:19.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:19.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:19 smithi017 ceph-mon[103001]: pgmap v535: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 240 KiB/s wr, 743 op/s 2024-09-13T14:32:19.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:19 smithi017 ceph-mon[107556]: pgmap v535: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 240 KiB/s wr, 743 op/s 2024-09-13T14:32:19.739 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:19 smithi189 ceph-mon[90992]: pgmap v535: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 240 KiB/s wr, 743 op/s 2024-09-13T14:32:19.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:19.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:19.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:19.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:19.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:19.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:20.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:20.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:20.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:20.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:20.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:20.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:20.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:20.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:20.571 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:20.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:20.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:20.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:20.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:20.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:32:20.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:20.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:20.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:20.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:20.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:21.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:21.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:21.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:21.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:21.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:21.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:21.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:21.435 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:32:21.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:21.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:21.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[103001]: pgmap v536: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:32:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3627970647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[107556]: pgmap v536: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:32:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3627970647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:21.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:21 smithi189 ceph-mon[90992]: pgmap v536: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:32:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3627970647' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:32:21.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:21.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:21.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:21.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:21.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:22.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:22.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:22.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:22.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:22.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:22.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:22.372 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:22.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:32:22.465 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:32:22.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:22.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:22.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:22.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:22.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[103001]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1702686882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[107556]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1702686882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:32:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:22 smithi189 ceph-mon[90992]: osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:32:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1702686882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:32:22.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:22.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:22.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:22.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:22.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:23.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:23.060 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:23.162 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:23.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:23.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:23.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:23.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:23.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:23.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:23.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:23.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:23.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:23.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:23.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:23 smithi189 ceph-mon[90992]: pgmap v538: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:32:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:23 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:32:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:23 smithi189 ceph-mon[90992]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:32:23.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:23.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:23.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[103001]: pgmap v538: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:32:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:32:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[103001]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:32:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[107556]: pgmap v538: 99 pgs: 99 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:32:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:32:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:23 smithi017 ceph-mon[107556]: osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:32:23.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:24.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:24.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:24.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:24.261 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:24.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:24.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:24.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:24.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:24.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:24.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:24.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:24.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:24.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:24.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:24.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:24.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:25.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:25.076 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:25.081 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:25.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:25.128 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:32:25.128 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:32:25.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:25.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:25.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:25.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:25.367 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.394 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:25.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:25.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:25.519 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:25.621 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:25.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:25.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:25.722 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:25 smithi017 ceph-mon[103001]: pgmap v540: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:32:25.723 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:25 smithi017 ceph-mon[107556]: pgmap v540: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:32:25.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:25 smithi189 ceph-mon[90992]: pgmap v540: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:32:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:25.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:25.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:25.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:25.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:26.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:26.058 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:26.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:26.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:26.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:26.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:26.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:26.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:26.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:32:26.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:26.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:26.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:26.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:26.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:26.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:26.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:26.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:32:26.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:27.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:27.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:27.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:27.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:27.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:27.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:27.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:27.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:27.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:27.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:27] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:32:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[103001]: pgmap v541: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T14:32:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[107556]: pgmap v541: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T14:32:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:27.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:27 smithi189 ceph-mon[90992]: pgmap v541: 99 pgs: 99 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T14:32:27.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:27.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:27.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:27.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:27.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:27.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:28.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:28.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:28.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:28.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:28.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:28.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:28.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:28.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:28.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:28.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:28.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:28.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:28.800 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:28.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:28.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:28.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:29.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:29.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:29.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:29.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:29.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:29.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:29.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:29.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:29.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:29.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:29 smithi017 ceph-mon[103001]: pgmap v542: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-09-13T14:32:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:29 smithi017 ceph-mon[107556]: pgmap v542: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-09-13T14:32:29.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:29 smithi189 ceph-mon[90992]: pgmap v542: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-09-13T14:32:29.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:29.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:29.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:29.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:30.005 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:30.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:30.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:30.167 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:30.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:30.297 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:30.345 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:30.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:30.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:30.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:30.566 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:30.598 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:32:30.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:30.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2372975179' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:30 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:30.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:30.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2372975179' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:30 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2372975179' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:30 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:32:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:30.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:30.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:31.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:31.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:31.224 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:31.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:31.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:31.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:31.489 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:31.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:31.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:31.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:31 smithi189 ceph-mon[90992]: pgmap v543: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:32:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:31 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:32:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:31 smithi189 ceph-mon[90992]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:32:31.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:31.774 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:31.777 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:31.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:31.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[103001]: pgmap v543: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:32:31.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:32:31.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[103001]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:32:31.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[107556]: pgmap v543: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:32:31.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:32:31.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:31 smithi017 ceph-mon[107556]: osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:32:31.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:31.937 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:31.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:32.039 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:32.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:32.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:32.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:32.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:32.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:32.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:32.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:32.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:32.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:32.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:32.600 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:32:32.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:32:32.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:32.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:32.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:32.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:32.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:32.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:33.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:33.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:33.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:33.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:32:33.172 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:32:33.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:32:33.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:32:33.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:33.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:33.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:32:33.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:32:33.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:32:33.272 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:32:33.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:32:33.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:32:33.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:33.337 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:32:33.337 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:32:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.360 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:33.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:33.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:32:33.366 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:32:33.404 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:32:33.404 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:32:33.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:33.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:33.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:33.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:33.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[103001]: pgmap v545: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-13T14:32:33.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3367367118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:33.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[107556]: pgmap v545: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-13T14:32:33.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3367367118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:33.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:33 smithi189 ceph-mon[90992]: pgmap v545: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-13T14:32:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3367367118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:33.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:33.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:33.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:33.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:34.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:34.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:34.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:34.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:34.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:34.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:34.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:34.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:34.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:34.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:34.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:34.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:34.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:35.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:35.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:35.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:35.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:35.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:35.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:35.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:35.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:35.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:35.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:35.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:35.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:35 smithi017 ceph-mon[103001]: pgmap v546: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:32:35.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:35 smithi017 ceph-mon[107556]: pgmap v546: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:32:35.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:35.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:35.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:35.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:35 smithi189 ceph-mon[90992]: pgmap v546: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:32:35.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:35.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:35.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:35.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:36.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:36.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:36.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:36.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:36.405 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:36.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:36.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:36.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150941 6.8 e194/173 6:1fe0e9a1:::benchmark_data_smithi017_185083_object26329:head v 194'9462, mlcod=194'9459)", 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.542078+0000", 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.017272081000000002, 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017299232000000001, 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542078+0000", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542078+0000", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542078+0000", 2024-09-13T14:32:36.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3300000000000002e-07 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542084+0000", 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1200000000000001e-06 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542093+0000", 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.542099+0000", 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9599999999999997e-06 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551049+0000", 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089499230000000003 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551069+0000", 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0086999999999999e-05 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.608 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150947 6.e e194/173 6:76369545:::benchmark_data_smithi017_185083_object26335:head v 194'9484, mlcod=194'9479)", 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.548330+0000", 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011020082, 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011101804999999999, 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548330+0000", 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548330+0000", 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548331+0000", 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6099999999999999e-07 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548337+0000", 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548349+0000", 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1477999999999999e-05 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.548356+0000", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3429999999999998e-06 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.553368+0000", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050117809999999999 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.553385+0000", 2024-09-13T14:32:36.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6881e-05 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150948 6.e e194/173 6:7eea3855:::benchmark_data_smithi017_185083_object26336:head v 194'9485, mlcod=194'9479)", 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.551719+0000", 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0076305510000000002, 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077648810000000004, 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551719+0000", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551719+0000", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551720+0000", 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7400000000000002e-07 2024-09-13T14:32:36.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551726+0000", 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1829999999999999e-06 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551736+0000", 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6609999999999995e-06 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.551742+0000", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6509999999999998e-06 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.558317+0000", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065752329999999998 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.558364+0000", 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6866000000000002e-05 2024-09-13T14:32:36.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150950 6.8 e194/173 6:1560277b:::benchmark_data_smithi017_185083_object26338:head v 194'9463, mlcod=194'9460)", 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.555276+0000", 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0040740689999999996, 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042591369999999996, 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555276+0000", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555276+0000", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555277+0000", 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4600000000000004e-07 2024-09-13T14:32:36.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555282+0000", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0039999999999999e-06 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555295+0000", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3028e-05 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555301+0000", 2024-09-13T14:32:36.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9090000000000003e-06 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555559+0000", 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025795899999999999 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.555607+0000", 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7660000000000001e-05 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150953 6.8 e194/173)", 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.556540+0000", 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0028094740000000002, 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030407030000000001, 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556540+0000", 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556540+0000", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556541+0000", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8299999999999996e-07 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556547+0000", 2024-09-13T14:32:36.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6300000000000003e-06 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556561+0000", 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4226000000000001e-05 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556568+0000", 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7490000000000001e-06 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:150951 6.1 e194/173 6:8953f522:::benchmark_data_smithi017_185083_object26339:head v 194'9458, mlcod=194'9456)", 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:32:36.556868+0000", 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002481965, 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027539639999999998, 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556868+0000", 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556868+0000", 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556869+0000", 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.99e-07 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:32:36.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556874+0000", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8100000000000003e-06 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556896+0000", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1379e-05 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556902+0000", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1999999999999999e-06 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:32:36.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556961+0000", 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9261000000000001e-05 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:32:36.556980+0000", 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9021e-05 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 6 2024-09-13T14:32:36.627 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:32:36.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:36.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:36.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:36.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:36.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:36.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:36.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:36.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:37.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:37.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:37.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:37.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:37.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:37.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:37.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:37.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:37.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:37.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:37] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:32:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:37 smithi017 ceph-mon[103001]: pgmap v547: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:32:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:37 smithi017 ceph-mon[107556]: pgmap v547: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:32:37.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:37 smithi189 ceph-mon[90992]: pgmap v547: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-09-13T14:32:37.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:37.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:37.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:37.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:37.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:38.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:38.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:38.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:38.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:38.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:32:38.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:38.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:38.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:38.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:38.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:38.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:38.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:38.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:38.901 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:38.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:38.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:38.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:39.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:39.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:39.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:39.240 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:39.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:39.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:32:39.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:39.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:39.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:39.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:39.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:39 smithi017 ceph-mon[103001]: pgmap v548: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:32:39.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:39 smithi017 ceph-mon[107556]: pgmap v548: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:32:39.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:39.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:39.783 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:39.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:39.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:39.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:39.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:39.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:39 smithi189 ceph-mon[90992]: pgmap v548: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-13T14:32:40.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:40.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:40.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:40.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:40.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:40.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:40.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:40.487 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:40.516 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:40.535 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:32:40.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3802534183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:40.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:40.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3802534183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:40.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:40.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:40.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:40.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:40.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:40.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:40.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:40.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3802534183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:32:41.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:41.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:41.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:41.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:41.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:41.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:41.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:41.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:41.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:41.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:41.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:41.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:41.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:41.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[103001]: pgmap v549: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T14:32:41.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:32:41.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:32:41.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:41.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[103001]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:32:41.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[107556]: pgmap v549: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T14:32:41.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:32:41.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:32:41.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:41.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:41 smithi017 ceph-mon[107556]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:32:41.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:41.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:41.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:41.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:41.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:41 smithi189 ceph-mon[90992]: pgmap v549: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-13T14:32:42.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:41 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:41 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:41 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:32:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:41 smithi189 ceph-mon[90992]: osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:32:42.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:42.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:42.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:42.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:42.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:42.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:42.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:42.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:42.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:42.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:42.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:42.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:42.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:42.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:42.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:43.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:43.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:43.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[103001]: pgmap v551: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-09-13T14:32:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[107556]: pgmap v551: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-09-13T14:32:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:43.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:43.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:43.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:43.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:43 smithi189 ceph-mon[90992]: pgmap v551: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-09-13T14:32:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:43.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:43.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:43.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:43.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:43.786 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:43.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:43.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:43.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:43.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:44.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:44.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:44.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:44.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:44.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:44.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:44.446 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:44.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:44.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:44.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:44.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:44.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:44.758 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:44.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:44.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:44.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:44.934 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:32:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:32:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:32:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:32:44.935 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:32:44.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:45.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:45.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:45.044 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:32:45.045 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:32:45.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:45.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:45.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.272 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:45.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:45.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:45 smithi017 ceph-mon[103001]: pgmap v552: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:32:45.346 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:45 smithi017 ceph-mon[107556]: pgmap v552: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:32:45.373 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.402 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:45.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:45.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:45.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:45 smithi189 ceph-mon[90992]: pgmap v552: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T14:32:45.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:45.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:45.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:45.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:45.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:45.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:45.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:45.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:46.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:46.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:46.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:32:46.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:46.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:46.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:32:46.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:46.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:46 smithi017 ceph-mon[103001]: from='client.45395 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:46.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:46 smithi017 ceph-mon[107556]: from='client.45395 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:46.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:46.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:46 smithi189 ceph-mon[90992]: from='client.45395 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:32:46.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:46.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:46.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:46.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:46.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:46.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:46.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:46.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:46.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:47.005 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:47.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:47.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:47 smithi017 ceph-mon[103001]: pgmap v553: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:32:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:47 smithi017 ceph-mon[107556]: pgmap v553: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:32:47.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:47.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:47.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:47.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:47.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:47 smithi189 ceph-mon[90992]: pgmap v553: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:32:47.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:47.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:47.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:47.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:47] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-13T14:32:47.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:47.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:47.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:47.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:47.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:47.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:48.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:48.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:48.125 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":552,"stamp":"2024-09-13T14:32:43.874441+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6132668389,"num_objects":93533,"num_object_clones":0,"num_object_copies":280599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93533,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":156622,"num_write_kb":7987600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132147,"ondisk_log_size":132147,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20528340,"kb_used_data":18110468,"kb_used_omap":35,"kb_used_meta":2417564,"kb_avail":729432876,"statfs":{"total":767960285184,"available":746939265024,"internally_reserved":0,"allocated":18545119232,"data_stored":18537037113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":2475586553},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,7,5,11,7,9,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-570032128,"num_objects":-8698,"num_object_clones":0,"num_object_copies":-26094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8698,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002816"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":255,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479414+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:32:30.479414+0000","last_peered":"2024-09-13T14:32:30.479414+0000","last_clean":"2024-09-13T14:32:30.479414+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:32:30.479414+0000","last_undegraded":"2024-09-13T14:32:30.479414+0000","last_fullsized":"2024-09-13T14:32:30.479414+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481704+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:32:30.481704+0000","last_peered":"2024-09-13T14:32:30.481704+0000","last_clean":"2024-09-13T14:32:30.481704+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:32:30.481704+0000","last_undegraded":"2024-09-13T14:32:30.481704+0000","last_fullsized":"2024-09-13T14:32:30.481704+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491188+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:32:30.491188+0000","last_peered":"2024-09-13T14:32:30.491188+0000","last_clean":"2024-09-13T14:32:30.491188+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:32:30.491188+0000","last_undegraded":"2024-09-13T14:32:30.491188+0000","last_fullsized":"2024-09-13T14:32:30.491188+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546086+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:32:40.546086+0000","last_peered":"2024-09-13T14:32:40.546086+0000","last_clean":"2024-09-13T14:32:40.546086+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:32:40.546086+0000","last_undegraded":"2024-09-13T14:32:40.546086+0000","last_fullsized":"2024-09-13T14:32:40.546086+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":254,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545791+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:32:40.545791+0000","last_peered":"2024-09-13T14:32:40.545791+0000","last_clean":"2024-09-13T14:32:40.545791+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:32:40.545791+0000","last_undegraded":"2024-09-13T14:32:40.545791+0000","last_fullsized":"2024-09-13T14:32:40.545791+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":142,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482240+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:32:30.482240+0000","last_peered":"2024-09-13T14:32:30.482240+0000","last_clean":"2024-09-13T14:32:30.482240+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:32:30.482240+0000","last_undegraded":"2024-09-13T14:32:30.482240+0000","last_fullsized":"2024-09-13T14:32:30.482240+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":255,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481963+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:32:30.481963+0000","last_peered":"2024-09-13T14:32:30.481963+0000","last_clean":"2024-09-13T14:32:30.481963+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:32:30.481963+0000","last_undegraded":"2024-09-13T14:32:30.481963+0000","last_fullsized":"2024-09-13T14:32:30.481963+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":167,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.478475+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:32:30.478475+0000","last_peered":"2024-09-13T14:32:30.478475+0000","last_clean":"2024-09-13T14:32:30.478475+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:32:30.478475+0000","last_undegraded":"2024-09-13T14:32:30.478475+0000","last_fullsized":"2024-09-13T14:32:30.478475+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"195'9612","reported_seq":9642,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.963275+0000","last_change":"2024-09-13T14:30:27.500678+0000","last_active":"2024-09-13T14:32:42.963275+0000","last_peered":"2024-09-13T14:32:42.963275+0000","last_clean":"2024-09-13T14:32:42.963275+0000","last_became_active":"2024-09-13T14:30:27.500235+0000","last_became_peered":"2024-09-13T14:30:27.500235+0000","last_unstale":"2024-09-13T14:32:42.963275+0000","last_undegraded":"2024-09-13T14:32:42.963275+0000","last_fullsized":"2024-09-13T14:32:42.963275+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9612,"log_dups_size":0,"ondisk_log_size":9612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:11.965913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375552,"num_objects":5682,"num_object_clones":0,"num_object_copies":17046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9612,"num_write_kb":489408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482584+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:32:30.482584+0000","last_peered":"2024-09-13T14:32:30.482584+0000","last_clean":"2024-09-13T14:32:30.482584+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:32:30.482584+0000","last_undegraded":"2024-09-13T14:32:30.482584+0000","last_fullsized":"2024-09-13T14:32:30.482584+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490813+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:32:30.490813+0000","last_peered":"2024-09-13T14:32:30.490813+0000","last_clean":"2024-09-13T14:32:30.490813+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:32:30.490813+0000","last_undegraded":"2024-09-13T14:32:30.490813+0000","last_fullsized":"2024-09-13T14:32:30.490813+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"194'9458","reported_seq":9511,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:39.289306+0000","last_change":"2024-09-13T14:30:27.502589+0000","last_active":"2024-09-13T14:32:39.289306+0000","last_peered":"2024-09-13T14:32:39.289306+0000","last_clean":"2024-09-13T14:32:39.289306+0000","last_became_active":"2024-09-13T14:30:27.502327+0000","last_became_peered":"2024-09-13T14:30:27.502327+0000","last_unstale":"2024-09-13T14:32:39.289306+0000","last_undegraded":"2024-09-13T14:32:39.289306+0000","last_fullsized":"2024-09-13T14:32:39.289306+0000","mapping_epoch":173,"log_start":"179'2900","ondisk_log_start":"179'2900","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:12.990070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390725632,"num_objects":5962,"num_object_clones":0,"num_object_copies":17886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9458,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482778+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:32:30.482778+0000","last_peered":"2024-09-13T14:32:30.482778+0000","last_clean":"2024-09-13T14:32:30.482778+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:32:30.482778+0000","last_undegraded":"2024-09-13T14:32:30.482778+0000","last_fullsized":"2024-09-13T14:32:30.482778+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488353+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:32:30.488353+0000","last_peered":"2024-09-13T14:32:30.488353+0000","last_clean":"2024-09-13T14:32:30.488353+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:32:30.488353+0000","last_undegraded":"2024-09-13T14:32:30.488353+0000","last_fullsized":"2024-09-13T14:32:30.488353+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"195'9801","reported_seq":9831,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.933249+0000","last_change":"2024-09-13T14:30:27.500921+0000","last_active":"2024-09-13T14:32:42.933249+0000","last_peered":"2024-09-13T14:32:42.933249+0000","last_clean":"2024-09-13T14:32:42.933249+0000","last_became_active":"2024-09-13T14:30:27.500457+0000","last_became_peered":"2024-09-13T14:30:27.500457+0000","last_unstale":"2024-09-13T14:32:42.933249+0000","last_undegraded":"2024-09-13T14:32:42.933249+0000","last_fullsized":"2024-09-13T14:32:42.933249+0000","mapping_epoch":173,"log_start":"178'2300","ondisk_log_start":"178'2300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:09.832130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379912192,"num_objects":5797,"num_object_clones":0,"num_object_copies":17391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9801,"num_write_kb":499136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":143,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482711+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:32:30.482711+0000","last_peered":"2024-09-13T14:32:30.482711+0000","last_clean":"2024-09-13T14:32:30.482711+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:32:30.482711+0000","last_undegraded":"2024-09-13T14:32:30.482711+0000","last_fullsized":"2024-09-13T14:32:30.482711+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479720+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:32:30.479720+0000","last_peered":"2024-09-13T14:32:30.479720+0000","last_clean":"2024-09-13T14:32:30.479720+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:32:30.479720+0000","last_undegraded":"2024-09-13T14:32:30.479720+0000","last_fullsized":"2024-09-13T14:32:30.479720+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"195'9720","reported_seq":9776,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.532312+0000","last_change":"2024-09-13T14:30:27.501287+0000","last_active":"2024-09-13T14:32:42.532312+0000","last_peered":"2024-09-13T14:32:42.532312+0000","last_clean":"2024-09-13T14:32:42.532312+0000","last_became_active":"2024-09-13T14:30:27.500947+0000","last_became_peered":"2024-09-13T14:30:27.500947+0000","last_unstale":"2024-09-13T14:32:42.532312+0000","last_undegraded":"2024-09-13T14:32:42.532312+0000","last_fullsized":"2024-09-13T14:32:42.532312+0000","mapping_epoch":173,"log_start":"177'600","ondisk_log_start":"177'600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9120,"log_dups_size":0,"ondisk_log_size":9120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:44:23.541886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378929152,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9720,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491651+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:32:30.491651+0000","last_peered":"2024-09-13T14:32:30.491651+0000","last_clean":"2024-09-13T14:32:30.491651+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:32:30.491651+0000","last_undegraded":"2024-09-13T14:32:30.491651+0000","last_fullsized":"2024-09-13T14:32:30.491651+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":254,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546563+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:32:40.546563+0000","last_peered":"2024-09-13T14:32:40.546563+0000","last_clean":"2024-09-13T14:32:40.546563+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:32:40.546563+0000","last_undegraded":"2024-09-13T14:32:40.546563+0000","last_fullsized":"2024-09-13T14:32:40.546563+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"195'9850","reported_seq":9880,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.976753+0000","last_change":"2024-09-13T14:30:27.501830+0000","last_active":"2024-09-13T14:32:42.976753+0000","last_peered":"2024-09-13T14:32:42.976753+0000","last_clean":"2024-09-13T14:32:42.976753+0000","last_became_active":"2024-09-13T14:30:27.501398+0000","last_became_peered":"2024-09-13T14:30:27.501398+0000","last_unstale":"2024-09-13T14:32:42.976753+0000","last_undegraded":"2024-09-13T14:32:42.976753+0000","last_fullsized":"2024-09-13T14:32:42.976753+0000","mapping_epoch":173,"log_start":"178'2500","ondisk_log_start":"178'2500","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:52.275184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384172032,"num_objects":5862,"num_object_clones":0,"num_object_copies":17586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9850,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":306,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482113+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:32:30.482113+0000","last_peered":"2024-09-13T14:32:30.482113+0000","last_clean":"2024-09-13T14:32:30.482113+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:32:30.482113+0000","last_undegraded":"2024-09-13T14:32:30.482113+0000","last_fullsized":"2024-09-13T14:32:30.482113+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":253,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479416+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:32:30.479416+0000","last_peered":"2024-09-13T14:32:30.479416+0000","last_clean":"2024-09-13T14:32:30.479416+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:32:30.479416+0000","last_undegraded":"2024-09-13T14:32:30.479416+0000","last_fullsized":"2024-09-13T14:32:30.479416+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488788+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:32:30.488788+0000","last_peered":"2024-09-13T14:32:30.488788+0000","last_clean":"2024-09-13T14:32:30.488788+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:32:30.488788+0000","last_undegraded":"2024-09-13T14:32:30.488788+0000","last_fullsized":"2024-09-13T14:32:30.488788+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"195'9722","reported_seq":9777,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:41.208888+0000","last_change":"2024-09-13T14:30:27.505193+0000","last_active":"2024-09-13T14:32:41.208888+0000","last_peered":"2024-09-13T14:32:41.208888+0000","last_clean":"2024-09-13T14:32:41.208888+0000","last_became_active":"2024-09-13T14:30:27.504909+0000","last_became_peered":"2024-09-13T14:30:27.504909+0000","last_unstale":"2024-09-13T14:32:41.208888+0000","last_undegraded":"2024-09-13T14:32:41.208888+0000","last_fullsized":"2024-09-13T14:32:41.208888+0000","mapping_epoch":173,"log_start":"177'300","ondisk_log_start":"177'300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9422,"log_dups_size":0,"ondisk_log_size":9422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:40.244032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386793472,"num_objects":5902,"num_object_clones":0,"num_object_copies":17706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9722,"num_write_kb":499968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":53,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490771+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:32:30.490771+0000","last_peered":"2024-09-13T14:32:30.490771+0000","last_clean":"2024-09-13T14:32:30.490771+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:32:30.490771+0000","last_undegraded":"2024-09-13T14:32:30.490771+0000","last_fullsized":"2024-09-13T14:32:30.490771+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.480184+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:32:30.480184+0000","last_peered":"2024-09-13T14:32:30.480184+0000","last_clean":"2024-09-13T14:32:30.480184+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:32:30.480184+0000","last_undegraded":"2024-09-13T14:32:30.480184+0000","last_fullsized":"2024-09-13T14:32:30.480184+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489252+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:32:30.489252+0000","last_peered":"2024-09-13T14:32:30.489252+0000","last_clean":"2024-09-13T14:32:30.489252+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:32:30.489252+0000","last_undegraded":"2024-09-13T14:32:30.489252+0000","last_fullsized":"2024-09-13T14:32:30.489252+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"194'9614","reported_seq":9667,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:39.268452+0000","last_change":"2024-09-13T14:30:27.503642+0000","last_active":"2024-09-13T14:32:39.268452+0000","last_peered":"2024-09-13T14:32:39.268452+0000","last_clean":"2024-09-13T14:32:39.268452+0000","last_became_active":"2024-09-13T14:30:27.503347+0000","last_became_peered":"2024-09-13T14:30:27.503347+0000","last_unstale":"2024-09-13T14:32:39.268452+0000","last_undegraded":"2024-09-13T14:32:39.268452+0000","last_fullsized":"2024-09-13T14:32:39.268452+0000","mapping_epoch":173,"log_start":"179'3000","ondisk_log_start":"179'3000","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:40.203221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390594560,"num_objects":5960,"num_object_clones":0,"num_object_copies":17880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9614,"num_write_kb":498368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488419+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:32:30.488419+0000","last_peered":"2024-09-13T14:32:30.488419+0000","last_clean":"2024-09-13T14:32:30.488419+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:32:30.488419+0000","last_undegraded":"2024-09-13T14:32:30.488419+0000","last_fullsized":"2024-09-13T14:32:30.488419+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489889+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:32:30.489889+0000","last_peered":"2024-09-13T14:32:30.489889+0000","last_clean":"2024-09-13T14:32:30.489889+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:32:30.489889+0000","last_undegraded":"2024-09-13T14:32:30.489889+0000","last_fullsized":"2024-09-13T14:32:30.489889+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"195'9779","reported_seq":9810,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.911360+0000","last_change":"2024-09-13T14:30:27.499796+0000","last_active":"2024-09-13T14:32:42.911360+0000","last_peered":"2024-09-13T14:32:42.911360+0000","last_clean":"2024-09-13T14:32:42.911360+0000","last_became_active":"2024-09-13T14:30:27.499475+0000","last_became_peered":"2024-09-13T14:30:27.499475+0000","last_unstale":"2024-09-13T14:32:42.911360+0000","last_undegraded":"2024-09-13T14:32:42.911360+0000","last_fullsized":"2024-09-13T14:32:42.911360+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9779,"log_dups_size":0,"ondisk_log_size":9779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:07.838538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379322392,"num_objects":5789,"num_object_clones":0,"num_object_copies":17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5789,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9779,"num_write_kb":498113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490978+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:32:30.490978+0000","last_peered":"2024-09-13T14:32:30.490978+0000","last_clean":"2024-09-13T14:32:30.490978+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:32:30.490978+0000","last_undegraded":"2024-09-13T14:32:30.490978+0000","last_fullsized":"2024-09-13T14:32:30.490978+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":257,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.478091+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:32:30.478091+0000","last_peered":"2024-09-13T14:32:30.478091+0000","last_clean":"2024-09-13T14:32:30.478091+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:32:30.478091+0000","last_undegraded":"2024-09-13T14:32:30.478091+0000","last_fullsized":"2024-09-13T14:32:30.478091+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533516+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:32:40.533516+0000","last_peered":"2024-09-13T14:32:40.533516+0000","last_clean":"2024-09-13T14:32:40.533516+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:32:40.533516+0000","last_undegraded":"2024-09-13T14:32:40.533516+0000","last_fullsized":"2024-09-13T14:32:40.533516+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"195'9984","reported_seq":10014,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.996947+0000","last_change":"2024-09-13T14:30:27.505472+0000","last_active":"2024-09-13T14:32:42.996947+0000","last_peered":"2024-09-13T14:32:42.996947+0000","last_clean":"2024-09-13T14:32:42.996947+0000","last_became_active":"2024-09-13T14:30:27.505098+0000","last_became_peered":"2024-09-13T14:30:27.505098+0000","last_unstale":"2024-09-13T14:32:42.996947+0000","last_undegraded":"2024-09-13T14:32:42.996947+0000","last_fullsized":"2024-09-13T14:32:42.996947+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9984,"log_dups_size":0,"ondisk_log_size":9984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:45.218708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382074880,"num_objects":5830,"num_object_clones":0,"num_object_copies":17490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9984,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":297,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482338+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:32:30.482338+0000","last_peered":"2024-09-13T14:32:30.482338+0000","last_clean":"2024-09-13T14:32:30.482338+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:32:30.482338+0000","last_undegraded":"2024-09-13T14:32:30.482338+0000","last_fullsized":"2024-09-13T14:32:30.482338+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":760,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490944+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:32:30.490944+0000","last_peered":"2024-09-13T14:32:30.490944+0000","last_clean":"2024-09-13T14:32:30.490944+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:32:30.490944+0000","last_undegraded":"2024-09-13T14:32:30.490944+0000","last_fullsized":"2024-09-13T14:32:30.490944+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":210,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.477588+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:32:30.477588+0000","last_peered":"2024-09-13T14:32:30.477588+0000","last_clean":"2024-09-13T14:32:30.477588+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:32:30.477588+0000","last_undegraded":"2024-09-13T14:32:30.477588+0000","last_fullsized":"2024-09-13T14:32:30.477588+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":141,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481839+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:32:30.481839+0000","last_peered":"2024-09-13T14:32:30.481839+0000","last_clean":"2024-09-13T14:32:30.481839+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:32:30.481839+0000","last_undegraded":"2024-09-13T14:32:30.481839+0000","last_fullsized":"2024-09-13T14:32:30.481839+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"195'9889","reported_seq":9945,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.434295+0000","last_change":"2024-09-13T14:30:27.505328+0000","last_active":"2024-09-13T14:32:42.434295+0000","last_peered":"2024-09-13T14:32:42.434295+0000","last_clean":"2024-09-13T14:32:42.434295+0000","last_became_active":"2024-09-13T14:30:27.505015+0000","last_became_peered":"2024-09-13T14:30:27.505015+0000","last_unstale":"2024-09-13T14:32:42.434295+0000","last_undegraded":"2024-09-13T14:32:42.434295+0000","last_fullsized":"2024-09-13T14:32:42.434295+0000","mapping_epoch":173,"log_start":"178'1900","ondisk_log_start":"178'1900","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7989,"log_dups_size":0,"ondisk_log_size":7989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:42.013686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382664704,"num_objects":5839,"num_object_clones":0,"num_object_copies":17517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9889,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":89,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534352+0000","last_change":"2024-09-13T14:30:39.822827+0000","last_active":"2024-09-13T14:32:40.534352+0000","last_peered":"2024-09-13T14:32:40.534352+0000","last_clean":"2024-09-13T14:32:40.534352+0000","last_became_active":"2024-09-13T14:30:11.437578+0000","last_became_peered":"2024-09-13T14:30:11.437578+0000","last_unstale":"2024-09-13T14:32:40.534352+0000","last_undegraded":"2024-09-13T14:32:40.534352+0000","last_fullsized":"2024-09-13T14:32:40.534352+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.822747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.822747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:44.769254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":568,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482196+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:32:30.482196+0000","last_peered":"2024-09-13T14:32:30.482196+0000","last_clean":"2024-09-13T14:32:30.482196+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:32:30.482196+0000","last_undegraded":"2024-09-13T14:32:30.482196+0000","last_fullsized":"2024-09-13T14:32:30.482196+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":254,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546629+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:32:40.546629+0000","last_peered":"2024-09-13T14:32:40.546629+0000","last_clean":"2024-09-13T14:32:40.546629+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:32:40.546629+0000","last_undegraded":"2024-09-13T14:32:40.546629+0000","last_fullsized":"2024-09-13T14:32:40.546629+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545947+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:32:40.545947+0000","last_peered":"2024-09-13T14:32:40.545947+0000","last_clean":"2024-09-13T14:32:40.545947+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:32:40.545947+0000","last_undegraded":"2024-09-13T14:32:40.545947+0000","last_fullsized":"2024-09-13T14:32:40.545947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"195'9717","reported_seq":9747,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.980532+0000","last_change":"2024-09-13T14:30:27.500479+0000","last_active":"2024-09-13T14:32:42.980532+0000","last_peered":"2024-09-13T14:32:42.980532+0000","last_clean":"2024-09-13T14:32:42.980532+0000","last_became_active":"2024-09-13T14:30:27.500110+0000","last_became_peered":"2024-09-13T14:30:27.500110+0000","last_unstale":"2024-09-13T14:32:42.980532+0000","last_undegraded":"2024-09-13T14:32:42.980532+0000","last_fullsized":"2024-09-13T14:32:42.980532+0000","mapping_epoch":173,"log_start":"178'2200","ondisk_log_start":"178'2200","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:10.174744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373751808,"num_objects":5703,"num_object_clones":0,"num_object_copies":17109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9717,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":306,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481360+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:32:30.481360+0000","last_peered":"2024-09-13T14:32:30.481360+0000","last_clean":"2024-09-13T14:32:30.481360+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:32:30.481360+0000","last_undegraded":"2024-09-13T14:32:30.481360+0000","last_fullsized":"2024-09-13T14:32:30.481360+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":683,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.543413+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:32:40.543413+0000","last_peered":"2024-09-13T14:32:40.543413+0000","last_clean":"2024-09-13T14:32:40.543413+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:32:40.543413+0000","last_undegraded":"2024-09-13T14:32:40.543413+0000","last_fullsized":"2024-09-13T14:32:40.543413+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533722+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:32:40.533722+0000","last_peered":"2024-09-13T14:32:40.533722+0000","last_clean":"2024-09-13T14:32:40.533722+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:32:40.533722+0000","last_undegraded":"2024-09-13T14:32:40.533722+0000","last_fullsized":"2024-09-13T14:32:40.533722+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538164+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:32:40.538164+0000","last_peered":"2024-09-13T14:32:40.538164+0000","last_clean":"2024-09-13T14:32:40.538164+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:32:40.538164+0000","last_undegraded":"2024-09-13T14:32:40.538164+0000","last_fullsized":"2024-09-13T14:32:40.538164+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"194'9593","reported_seq":9647,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:39.258988+0000","last_change":"2024-09-13T14:30:27.501260+0000","last_active":"2024-09-13T14:32:39.258988+0000","last_peered":"2024-09-13T14:32:39.258988+0000","last_clean":"2024-09-13T14:32:39.258988+0000","last_became_active":"2024-09-13T14:30:27.500950+0000","last_became_peered":"2024-09-13T14:30:27.500950+0000","last_unstale":"2024-09-13T14:32:39.258988+0000","last_undegraded":"2024-09-13T14:32:39.258988+0000","last_fullsized":"2024-09-13T14:32:39.258988+0000","mapping_epoch":173,"log_start":"179'3000","ondisk_log_start":"179'3000","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:54.878021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389218304,"num_objects":5939,"num_object_clones":0,"num_object_copies":17817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9593,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490923+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:32:30.490923+0000","last_peered":"2024-09-13T14:32:30.490923+0000","last_clean":"2024-09-13T14:32:30.490923+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:32:30.490923+0000","last_undegraded":"2024-09-13T14:32:30.490923+0000","last_fullsized":"2024-09-13T14:32:30.490923+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":809,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491109+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:32:30.491109+0000","last_peered":"2024-09-13T14:32:30.491109+0000","last_clean":"2024-09-13T14:32:30.491109+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:32:30.491109+0000","last_undegraded":"2024-09-13T14:32:30.491109+0000","last_fullsized":"2024-09-13T14:32:30.491109+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488977+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:32:30.488977+0000","last_peered":"2024-09-13T14:32:30.488977+0000","last_clean":"2024-09-13T14:32:30.488977+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:32:30.488977+0000","last_undegraded":"2024-09-13T14:32:30.488977+0000","last_fullsized":"2024-09-13T14:32:30.488977+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534022+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:32:40.534022+0000","last_peered":"2024-09-13T14:32:40.534022+0000","last_clean":"2024-09-13T14:32:40.534022+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:32:40.534022+0000","last_undegraded":"2024-09-13T14:32:40.534022+0000","last_fullsized":"2024-09-13T14:32:40.534022+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"195'9493","reported_seq":9549,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.518843+0000","last_change":"2024-09-13T14:30:27.502045+0000","last_active":"2024-09-13T14:32:42.518843+0000","last_peered":"2024-09-13T14:32:42.518843+0000","last_clean":"2024-09-13T14:32:42.518843+0000","last_became_active":"2024-09-13T14:30:27.501711+0000","last_became_peered":"2024-09-13T14:30:27.501711+0000","last_unstale":"2024-09-13T14:32:42.518843+0000","last_undegraded":"2024-09-13T14:32:42.518843+0000","last_fullsized":"2024-09-13T14:32:42.518843+0000","mapping_epoch":173,"log_start":"177'400","ondisk_log_start":"177'400","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9093,"log_dups_size":0,"ondisk_log_size":9093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:21.043630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372310016,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9493,"num_write_kb":485568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":194,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479838+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:32:30.479838+0000","last_peered":"2024-09-13T14:32:30.479838+0000","last_clean":"2024-09-13T14:32:30.479838+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:32:30.479838+0000","last_undegraded":"2024-09-13T14:32:30.479838+0000","last_fullsized":"2024-09-13T14:32:30.479838+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491260+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:32:30.491260+0000","last_peered":"2024-09-13T14:32:30.491260+0000","last_clean":"2024-09-13T14:32:30.491260+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:32:30.491260+0000","last_undegraded":"2024-09-13T14:32:30.491260+0000","last_fullsized":"2024-09-13T14:32:30.491260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":854,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534173+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:32:40.534173+0000","last_peered":"2024-09-13T14:32:40.534173+0000","last_clean":"2024-09-13T14:32:40.534173+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:32:40.534173+0000","last_undegraded":"2024-09-13T14:32:40.534173+0000","last_fullsized":"2024-09-13T14:32:40.534173+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":233,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533309+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:32:40.533309+0000","last_peered":"2024-09-13T14:32:40.533309+0000","last_clean":"2024-09-13T14:32:40.533309+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:32:40.533309+0000","last_undegraded":"2024-09-13T14:32:40.533309+0000","last_fullsized":"2024-09-13T14:32:40.533309+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"195'9779","reported_seq":9835,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.414216+0000","last_change":"2024-09-13T14:30:27.500989+0000","last_active":"2024-09-13T14:32:42.414216+0000","last_peered":"2024-09-13T14:32:42.414216+0000","last_clean":"2024-09-13T14:32:42.414216+0000","last_became_active":"2024-09-13T14:30:27.500671+0000","last_became_peered":"2024-09-13T14:30:27.500671+0000","last_unstale":"2024-09-13T14:32:42.414216+0000","last_undegraded":"2024-09-13T14:32:42.414216+0000","last_fullsized":"2024-09-13T14:32:42.414216+0000","mapping_epoch":173,"log_start":"178'1800","ondisk_log_start":"178'1800","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7979,"log_dups_size":0,"ondisk_log_size":7979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:59.179207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390922240,"num_objects":5965,"num_object_clones":0,"num_object_copies":17895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9779,"num_write_kb":503808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":194,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479685+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:32:30.479685+0000","last_peered":"2024-09-13T14:32:30.479685+0000","last_clean":"2024-09-13T14:32:30.479685+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:32:30.479685+0000","last_undegraded":"2024-09-13T14:32:30.479685+0000","last_fullsized":"2024-09-13T14:32:30.479685+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534094+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:32:40.534094+0000","last_peered":"2024-09-13T14:32:40.534094+0000","last_clean":"2024-09-13T14:32:40.534094+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:32:40.534094+0000","last_undegraded":"2024-09-13T14:32:40.534094+0000","last_fullsized":"2024-09-13T14:32:40.534094+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":505,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482720+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:32:30.482720+0000","last_peered":"2024-09-13T14:32:30.482720+0000","last_clean":"2024-09-13T14:32:30.482720+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:32:30.482720+0000","last_undegraded":"2024-09-13T14:32:30.482720+0000","last_fullsized":"2024-09-13T14:32:30.482720+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546489+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:32:40.546489+0000","last_peered":"2024-09-13T14:32:40.546489+0000","last_clean":"2024-09-13T14:32:40.546489+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:32:40.546489+0000","last_undegraded":"2024-09-13T14:32:40.546489+0000","last_fullsized":"2024-09-13T14:32:40.546489+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"195'9866","reported_seq":9896,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:42.971503+0000","last_change":"2024-09-13T14:30:27.502612+0000","last_active":"2024-09-13T14:32:42.971503+0000","last_peered":"2024-09-13T14:32:42.971503+0000","last_clean":"2024-09-13T14:32:42.971503+0000","last_became_active":"2024-09-13T14:30:27.501659+0000","last_became_peered":"2024-09-13T14:30:27.501659+0000","last_unstale":"2024-09-13T14:32:42.971503+0000","last_undegraded":"2024-09-13T14:32:42.971503+0000","last_fullsized":"2024-09-13T14:32:42.971503+0000","mapping_epoch":173,"log_start":"178'2500","ondisk_log_start":"178'2500","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:47.542650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383254528,"num_objects":5848,"num_object_clones":0,"num_object_copies":17544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9866,"num_write_kb":502848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489054+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:32:30.489054+0000","last_peered":"2024-09-13T14:32:30.489054+0000","last_clean":"2024-09-13T14:32:30.489054+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:32:30.489054+0000","last_undegraded":"2024-09-13T14:32:30.489054+0000","last_fullsized":"2024-09-13T14:32:30.489054+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":254,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482188+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:32:30.482188+0000","last_peered":"2024-09-13T14:32:30.482188+0000","last_clean":"2024-09-13T14:32:30.482188+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:32:30.482188+0000","last_undegraded":"2024-09-13T14:32:30.482188+0000","last_fullsized":"2024-09-13T14:32:30.482188+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":588,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533469+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:32:40.533469+0000","last_peered":"2024-09-13T14:32:40.533469+0000","last_clean":"2024-09-13T14:32:40.533469+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:32:40.533469+0000","last_undegraded":"2024-09-13T14:32:40.533469+0000","last_fullsized":"2024-09-13T14:32:40.533469+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.477819+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:32:30.477819+0000","last_peered":"2024-09-13T14:32:30.477819+0000","last_clean":"2024-09-13T14:32:30.477819+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:32:30.477819+0000","last_undegraded":"2024-09-13T14:32:30.477819+0000","last_fullsized":"2024-09-13T14:32:30.477819+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"195'9838","reported_seq":9868,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:43.000466+0000","last_change":"2024-09-13T14:30:27.501009+0000","last_active":"2024-09-13T14:32:43.000466+0000","last_peered":"2024-09-13T14:32:43.000466+0000","last_clean":"2024-09-13T14:32:43.000466+0000","last_became_active":"2024-09-13T14:30:27.500481+0000","last_became_peered":"2024-09-13T14:30:27.500481+0000","last_unstale":"2024-09-13T14:32:43.000466+0000","last_undegraded":"2024-09-13T14:32:43.000466+0000","last_fullsized":"2024-09-13T14:32:43.000466+0000","mapping_epoch":173,"log_start":"178'2300","ondisk_log_start":"178'2300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:30.536488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390856704,"num_objects":5964,"num_object_clones":0,"num_object_copies":17892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9838,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":92,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479254+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:32:30.479254+0000","last_peered":"2024-09-13T14:32:30.479254+0000","last_clean":"2024-09-13T14:32:30.479254+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:32:30.479254+0000","last_undegraded":"2024-09-13T14:32:30.479254+0000","last_fullsized":"2024-09-13T14:32:30.479254+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":258,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545696+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:32:40.545696+0000","last_peered":"2024-09-13T14:32:40.545696+0000","last_clean":"2024-09-13T14:32:40.545696+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:32:40.545696+0000","last_undegraded":"2024-09-13T14:32:40.545696+0000","last_fullsized":"2024-09-13T14:32:40.545696+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":613,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.480031+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:32:30.480031+0000","last_peered":"2024-09-13T14:32:30.480031+0000","last_clean":"2024-09-13T14:32:30.480031+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:32:30.480031+0000","last_undegraded":"2024-09-13T14:32:30.480031+0000","last_fullsized":"2024-09-13T14:32:30.480031+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":142,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488245+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:32:30.488245+0000","last_peered":"2024-09-13T14:32:30.488245+0000","last_clean":"2024-09-13T14:32:30.488245+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:32:30.488245+0000","last_undegraded":"2024-09-13T14:32:30.488245+0000","last_fullsized":"2024-09-13T14:32:30.488245+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488859+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:32:30.488859+0000","last_peered":"2024-09-13T14:32:30.488859+0000","last_clean":"2024-09-13T14:32:30.488859+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:32:30.488859+0000","last_undegraded":"2024-09-13T14:32:30.488859+0000","last_fullsized":"2024-09-13T14:32:30.488859+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":165,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488876+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:32:30.488876+0000","last_peered":"2024-09-13T14:32:30.488876+0000","last_clean":"2024-09-13T14:32:30.488876+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:32:30.488876+0000","last_undegraded":"2024-09-13T14:32:30.488876+0000","last_fullsized":"2024-09-13T14:32:30.488876+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533148+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:32:40.533148+0000","last_peered":"2024-09-13T14:32:40.533148+0000","last_clean":"2024-09-13T14:32:40.533148+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:32:40.533148+0000","last_undegraded":"2024-09-13T14:32:40.533148+0000","last_fullsized":"2024-09-13T14:32:40.533148+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":162,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482559+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:32:30.482559+0000","last_peered":"2024-09-13T14:32:30.482559+0000","last_clean":"2024-09-13T14:32:30.482559+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:32:30.482559+0000","last_undegraded":"2024-09-13T14:32:30.482559+0000","last_fullsized":"2024-09-13T14:32:30.482559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491106+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:32:30.491106+0000","last_peered":"2024-09-13T14:32:30.491106+0000","last_clean":"2024-09-13T14:32:30.491106+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:32:30.491106+0000","last_undegraded":"2024-09-13T14:32:30.491106+0000","last_fullsized":"2024-09-13T14:32:30.491106+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":210,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.483756+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:32:30.483756+0000","last_peered":"2024-09-13T14:32:30.483756+0000","last_clean":"2024-09-13T14:32:30.483756+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:32:30.483756+0000","last_undegraded":"2024-09-13T14:32:30.483756+0000","last_fullsized":"2024-09-13T14:32:30.483756+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":253,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546128+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:32:40.546128+0000","last_peered":"2024-09-13T14:32:40.546128+0000","last_clean":"2024-09-13T14:32:40.546128+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:32:40.546128+0000","last_undegraded":"2024-09-13T14:32:40.546128+0000","last_fullsized":"2024-09-13T14:32:40.546128+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":255,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.477736+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:32:30.477736+0000","last_peered":"2024-09-13T14:32:30.477736+0000","last_clean":"2024-09-13T14:32:30.477736+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:32:30.477736+0000","last_undegraded":"2024-09-13T14:32:30.477736+0000","last_fullsized":"2024-09-13T14:32:30.477736+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479916+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:32:30.479916+0000","last_peered":"2024-09-13T14:32:30.479916+0000","last_clean":"2024-09-13T14:32:30.479916+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:32:30.479916+0000","last_undegraded":"2024-09-13T14:32:30.479916+0000","last_fullsized":"2024-09-13T14:32:30.479916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":273,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.477972+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:32:30.477972+0000","last_peered":"2024-09-13T14:32:30.477972+0000","last_clean":"2024-09-13T14:32:30.477972+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:32:30.477972+0000","last_undegraded":"2024-09-13T14:32:30.477972+0000","last_fullsized":"2024-09-13T14:32:30.477972+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545864+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:32:40.545864+0000","last_peered":"2024-09-13T14:32:40.545864+0000","last_clean":"2024-09-13T14:32:40.545864+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:32:40.545864+0000","last_undegraded":"2024-09-13T14:32:40.545864+0000","last_fullsized":"2024-09-13T14:32:40.545864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":245,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533831+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:32:40.533831+0000","last_peered":"2024-09-13T14:32:40.533831+0000","last_clean":"2024-09-13T14:32:40.533831+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:32:40.533831+0000","last_undegraded":"2024-09-13T14:32:40.533831+0000","last_fullsized":"2024-09-13T14:32:40.533831+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482671+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:32:30.482671+0000","last_peered":"2024-09-13T14:32:30.482671+0000","last_clean":"2024-09-13T14:32:30.482671+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:32:30.482671+0000","last_undegraded":"2024-09-13T14:32:30.482671+0000","last_fullsized":"2024-09-13T14:32:30.482671+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":178,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489279+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:32:30.489279+0000","last_peered":"2024-09-13T14:32:30.489279+0000","last_clean":"2024-09-13T14:32:30.489279+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:32:30.489279+0000","last_undegraded":"2024-09-13T14:32:30.489279+0000","last_fullsized":"2024-09-13T14:32:30.489279+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.537703+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:32:40.537703+0000","last_peered":"2024-09-13T14:32:40.537703+0000","last_clean":"2024-09-13T14:32:40.537703+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:32:40.537703+0000","last_undegraded":"2024-09-13T14:32:40.537703+0000","last_fullsized":"2024-09-13T14:32:40.537703+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":246,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546395+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:32:40.546395+0000","last_peered":"2024-09-13T14:32:40.546395+0000","last_clean":"2024-09-13T14:32:40.546395+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:32:40.546395+0000","last_undegraded":"2024-09-13T14:32:40.546395+0000","last_fullsized":"2024-09-13T14:32:40.546395+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488788+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:32:30.488788+0000","last_peered":"2024-09-13T14:32:30.488788+0000","last_clean":"2024-09-13T14:32:30.488788+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:32:30.488788+0000","last_undegraded":"2024-09-13T14:32:30.488788+0000","last_fullsized":"2024-09-13T14:32:30.488788+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491534+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:32:30.491534+0000","last_peered":"2024-09-13T14:32:30.491534+0000","last_clean":"2024-09-13T14:32:30.491534+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:32:30.491534+0000","last_undegraded":"2024-09-13T14:32:30.491534+0000","last_fullsized":"2024-09-13T14:32:30.491534+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489198+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:32:30.489198+0000","last_peered":"2024-09-13T14:32:30.489198+0000","last_clean":"2024-09-13T14:32:30.489198+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:32:30.489198+0000","last_undegraded":"2024-09-13T14:32:30.489198+0000","last_fullsized":"2024-09-13T14:32:30.489198+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":144,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491242+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:32:30.491242+0000","last_peered":"2024-09-13T14:32:30.491242+0000","last_clean":"2024-09-13T14:32:30.491242+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:32:30.491242+0000","last_undegraded":"2024-09-13T14:32:30.491242+0000","last_fullsized":"2024-09-13T14:32:30.491242+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":276,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.484068+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:32:30.484068+0000","last_peered":"2024-09-13T14:32:30.484068+0000","last_clean":"2024-09-13T14:32:30.484068+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:32:30.484068+0000","last_undegraded":"2024-09-13T14:32:30.484068+0000","last_fullsized":"2024-09-13T14:32:30.484068+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":262,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.478682+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:32:30.478682+0000","last_peered":"2024-09-13T14:32:30.478682+0000","last_clean":"2024-09-13T14:32:30.478682+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:32:30.478682+0000","last_undegraded":"2024-09-13T14:32:30.478682+0000","last_fullsized":"2024-09-13T14:32:30.478682+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488096+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:32:30.488096+0000","last_peered":"2024-09-13T14:32:30.488096+0000","last_clean":"2024-09-13T14:32:30.488096+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:32:30.488096+0000","last_undegraded":"2024-09-13T14:32:30.488096+0000","last_fullsized":"2024-09-13T14:32:30.488096+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":258,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546044+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:32:40.546044+0000","last_peered":"2024-09-13T14:32:40.546044+0000","last_clean":"2024-09-13T14:32:40.546044+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:32:40.546044+0000","last_undegraded":"2024-09-13T14:32:40.546044+0000","last_fullsized":"2024-09-13T14:32:40.546044+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6127878168,"num_objects":93505,"num_object_clones":0,"num_object_copies":280515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93505,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155715,"num_write_kb":7974977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18427359232,"data_stored":18427347016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130015,"ondisk_log_size":130015,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397759,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2537904,"kb_used_data":2232232,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91207248,"statfs":{"total":95995035648,"available":93396221952,"internally_reserved":0,"allocated":2285805568,"data_stored":2284783321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593987,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2174244,"kb_used_data":1868568,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91570908,"statfs":{"total":95995035648,"available":93768609792,"internally_reserved":0,"allocated":1913413632,"data_stored":1912407075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790217,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156576,"kb_used_data":1855064,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91588576,"statfs":{"total":95995035648,"available":93786701824,"internally_reserved":0,"allocated":1899585536,"data_stored":1898575310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986445,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179440,"kb_used_data":1869604,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91565712,"statfs":{"total":95995035648,"available":93763289088,"internally_reserved":0,"allocated":1914474496,"data_stored":1913456275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697002,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4175056,"kb_used_data":3851400,"kb_used_omap":4,"kb_used_meta":323579,"kb_avail":89570096,"statfs":{"total":95995035648,"available":91719778304,"internally_reserved":0,"allocated":3943833600,"data_stored":3942831473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":331344971},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860529,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3349796,"kb_used_data":3056536,"kb_used_omap":8,"kb_used_meta":293175,"kb_avail":90395356,"statfs":{"total":95995035648,"available":92564844544,"internally_reserved":0,"allocated":3129892864,"data_stored":3128883209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":300212044},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024056,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179292,"kb_used_data":1895700,"kb_used_omap":7,"kb_used_meta":283512,"kb_avail":91565860,"statfs":{"total":95995035648,"available":93763440640,"internally_reserved":0,"allocated":1941196800,"data_stored":1940190120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":290316727},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187591,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776032,"kb_used_data":1481364,"kb_used_omap":7,"kb_used_meta":294648,"kb_avail":91969120,"statfs":{"total":95995035648,"available":94176378880,"internally_reserved":0,"allocated":1516916736,"data_stored":1515910330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":301719883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,1,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1505562624,"data_stored":1505558552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1921388544,"data_stored":1921384472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3108306944,"data_stored":3108306944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3912761344,"data_stored":3912761344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1906704384,"data_stored":1906704384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1891504128,"data_stored":1891500056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1904869376,"data_stored":1904869376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2276261888,"data_stored":2276261888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:32:48.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:32:48.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:48.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:48.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:48.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:48.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:48.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:48.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:48.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:48.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:48.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:48.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:48.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:48.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:48.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:48.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:49.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:49.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:49.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:49.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:49.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:49.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:49.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:49.453 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:49.454 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:49.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:49 smithi017 ceph-mon[103001]: pgmap v554: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:32:49.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:49 smithi017 ceph-mon[107556]: pgmap v554: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:32:49.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:49.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:49 smithi189 ceph-mon[90992]: pgmap v554: 99 pgs: 99 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:32:49.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:49.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:49.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:49.777 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:32:49.777 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:32:49.819 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:32:49.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:32:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:49.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:49.861 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:32:49.862 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:32:49.899 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:32:49.899 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:32:49.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:49.928 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:32:49.928 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:32:49.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:49.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:49.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:49.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:32:49.959 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:32:49.991 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:32:49.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:32:50.023 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:32:50.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:32:50.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:50.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:50.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:50.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:50.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.342 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:50.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:50.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2979237519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2979237519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:50.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:32:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:32:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2979237519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:32:50.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:50.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:50.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:50.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:50.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:50.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:50.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:50.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:51.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:51.055 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:51.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:51.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:51.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:51.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:51.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:51 smithi017 ceph-mon[103001]: pgmap v555: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T14:32:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:51 smithi017 ceph-mon[107556]: pgmap v555: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T14:32:51.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:51.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:51 smithi189 ceph-mon[90992]: pgmap v555: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T14:32:51.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:51.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:51.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:51.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:51.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:51.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:51.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:51.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:51.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:52.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:52.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:52.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:52.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:52.355 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:52.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:52.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:52.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:52.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:52.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:52.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:52.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:52.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:52.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:52.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:53.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:53.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.148 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:53.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:53.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:53.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:53.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:53.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:53 smithi017 ceph-mon[103001]: pgmap v556: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-09-13T14:32:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:53 smithi017 ceph-mon[107556]: pgmap v556: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-09-13T14:32:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:53 smithi189 ceph-mon[90992]: pgmap v556: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-09-13T14:32:53.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:53.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:53.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:53.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:53.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:53.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:53.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:32:53.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:53.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:53.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:53.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:54.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:54.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:54.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:54.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:54.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:54.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:54.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:54.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:54.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:54.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:54.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:54.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:54.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:54.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:54.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:54.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:55.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:55.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:55.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:55.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:55.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:55.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:55 smithi017 ceph-mon[103001]: pgmap v557: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:32:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:55 smithi017 ceph-mon[107556]: pgmap v557: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:32:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:55 smithi189 ceph-mon[90992]: pgmap v557: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2024-09-13T14:32:55.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:55.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:55.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:55.721 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:55.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:55.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:55.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:55.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:56.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:56.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:56.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:56.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:56.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:56.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:56.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:56.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:56.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:56.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:56.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:56.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:56.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:56.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:32:56.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:56.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:56.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:56.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:56.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:56.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:57.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:57.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:57.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:57.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:57.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:57.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:57.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:57.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:57.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:57.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:57.652 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:32:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:32:57] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:32:57.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:57 smithi017 ceph-mon[103001]: pgmap v558: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:32:57.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:57.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:57 smithi017 ceph-mon[107556]: pgmap v558: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:32:57.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:57.668 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:57.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:57 smithi189 ceph-mon[90992]: pgmap v558: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-09-13T14:32:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:57.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:57.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:57.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:57.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:57.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:58.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:32:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:58.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:32:58.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:58.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:58.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:58.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:58.455 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:32:58.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:32:58.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1023284659' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.533 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1023284659' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.533 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:32:58.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:58.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1023284659' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:32:58.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:32:58.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:32:58.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:32:58.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:58.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:32:58.985 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:32:58.985 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:32:59.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:59.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:59.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:59.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:59.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:32:59.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:32:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:32:59.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:32:59.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:32:59.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:59.643 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:32:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[103001]: pgmap v559: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:32:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[103001]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[107556]: pgmap v559: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:32:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:32:59 smithi017 ceph-mon[107556]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:32:59.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:32:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:59 smithi189 ceph-mon[90992]: pgmap v559: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-09-13T14:32:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:59 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:32:59.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:59 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:32:59.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:32:59 smithi189 ceph-mon[90992]: osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:32:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:32:59.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:32:59.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:32:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:32:59.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:00.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:00.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:00.148 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:33:00.153 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:33:00.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:00.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:00.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:00.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:00.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:00.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:00.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:00.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:00.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:00.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:00.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:00.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:01.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:01.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:01.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:01.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:01.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:01.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:01.440 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":560,"stamp":"2024-09-13T14:32:58.450811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5307635685,"num_objects":80944,"num_object_clones":0,"num_object_copies":242832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80944,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":169211,"num_write_kb":7987600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133136,"ondisk_log_size":133136,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18148688,"kb_used_data":15683536,"kb_used_omap":35,"kb_used_meta":2464732,"kb_avail":731812528,"statfs":{"total":767960285184,"available":749376028672,"internally_reserved":0,"allocated":16059940864,"data_stored":16051854791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":2523886585},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-687079424,"num_objects":-10484,"num_object_clones":0,"num_object_copies":-31452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10484,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.575247"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":257,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.551741+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:32:40.551741+0000","last_peered":"2024-09-13T14:32:40.551741+0000","last_clean":"2024-09-13T14:32:40.551741+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:32:40.551741+0000","last_undegraded":"2024-09-13T14:32:40.551741+0000","last_fullsized":"2024-09-13T14:32:40.551741+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481704+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:32:30.481704+0000","last_peered":"2024-09-13T14:32:30.481704+0000","last_clean":"2024-09-13T14:32:30.481704+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:32:30.481704+0000","last_undegraded":"2024-09-13T14:32:30.481704+0000","last_fullsized":"2024-09-13T14:32:30.481704+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491188+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:32:30.491188+0000","last_peered":"2024-09-13T14:32:30.491188+0000","last_clean":"2024-09-13T14:32:30.491188+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:32:30.491188+0000","last_undegraded":"2024-09-13T14:32:30.491188+0000","last_fullsized":"2024-09-13T14:32:30.491188+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546086+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:32:40.546086+0000","last_peered":"2024-09-13T14:32:40.546086+0000","last_clean":"2024-09-13T14:32:40.546086+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:32:40.546086+0000","last_undegraded":"2024-09-13T14:32:40.546086+0000","last_fullsized":"2024-09-13T14:32:40.546086+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":254,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545791+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:32:40.545791+0000","last_peered":"2024-09-13T14:32:40.545791+0000","last_clean":"2024-09-13T14:32:40.545791+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:32:40.545791+0000","last_undegraded":"2024-09-13T14:32:40.545791+0000","last_fullsized":"2024-09-13T14:32:40.545791+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":142,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482240+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:32:30.482240+0000","last_peered":"2024-09-13T14:32:30.482240+0000","last_clean":"2024-09-13T14:32:30.482240+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:32:30.482240+0000","last_undegraded":"2024-09-13T14:32:30.482240+0000","last_fullsized":"2024-09-13T14:32:30.482240+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":257,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.552104+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:32:40.552104+0000","last_peered":"2024-09-13T14:32:40.552104+0000","last_clean":"2024-09-13T14:32:40.552104+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:32:40.552104+0000","last_undegraded":"2024-09-13T14:32:40.552104+0000","last_fullsized":"2024-09-13T14:32:40.552104+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":167,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.478475+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:32:30.478475+0000","last_peered":"2024-09-13T14:32:30.478475+0000","last_clean":"2024-09-13T14:32:30.478475+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:32:30.478475+0000","last_undegraded":"2024-09-13T14:32:30.478475+0000","last_fullsized":"2024-09-13T14:32:30.478475+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"195'10372","reported_seq":10402,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.995319+0000","last_change":"2024-09-13T14:30:27.500678+0000","last_active":"2024-09-13T14:32:57.995319+0000","last_peered":"2024-09-13T14:32:57.995319+0000","last_clean":"2024-09-13T14:32:57.995319+0000","last_became_active":"2024-09-13T14:30:27.500235+0000","last_became_peered":"2024-09-13T14:30:27.500235+0000","last_unstale":"2024-09-13T14:32:57.995319+0000","last_undegraded":"2024-09-13T14:32:57.995319+0000","last_fullsized":"2024-09-13T14:32:57.995319+0000","mapping_epoch":173,"log_start":"177'300","ondisk_log_start":"177'300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:11.965913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568192,"num_objects":4922,"num_object_clones":0,"num_object_copies":14766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10372,"num_write_kb":489408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482584+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:32:30.482584+0000","last_peered":"2024-09-13T14:32:30.482584+0000","last_clean":"2024-09-13T14:32:30.482584+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:32:30.482584+0000","last_undegraded":"2024-09-13T14:32:30.482584+0000","last_fullsized":"2024-09-13T14:32:30.482584+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490813+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:32:30.490813+0000","last_peered":"2024-09-13T14:32:30.490813+0000","last_clean":"2024-09-13T14:32:30.490813+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:32:30.490813+0000","last_undegraded":"2024-09-13T14:32:30.490813+0000","last_fullsized":"2024-09-13T14:32:30.490813+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"195'10281","reported_seq":10337,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:54.362604+0000","last_change":"2024-09-13T14:30:27.502589+0000","last_active":"2024-09-13T14:32:54.362604+0000","last_peered":"2024-09-13T14:32:54.362604+0000","last_clean":"2024-09-13T14:32:54.362604+0000","last_became_active":"2024-09-13T14:30:27.502327+0000","last_became_peered":"2024-09-13T14:30:27.502327+0000","last_unstale":"2024-09-13T14:32:54.362604+0000","last_undegraded":"2024-09-13T14:32:54.362604+0000","last_fullsized":"2024-09-13T14:32:54.362604+0000","mapping_epoch":173,"log_start":"180'3700","ondisk_log_start":"180'3700","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6581,"log_dups_size":0,"ondisk_log_size":6581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:12.990070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336789504,"num_objects":5139,"num_object_clones":0,"num_object_copies":15417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10281,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482778+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:32:30.482778+0000","last_peered":"2024-09-13T14:32:30.482778+0000","last_clean":"2024-09-13T14:32:30.482778+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:32:30.482778+0000","last_undegraded":"2024-09-13T14:32:30.482778+0000","last_fullsized":"2024-09-13T14:32:30.482778+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488353+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:32:30.488353+0000","last_peered":"2024-09-13T14:32:30.488353+0000","last_clean":"2024-09-13T14:32:30.488353+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:32:30.488353+0000","last_undegraded":"2024-09-13T14:32:30.488353+0000","last_fullsized":"2024-09-13T14:32:30.488353+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"195'10596","reported_seq":10626,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.953863+0000","last_change":"2024-09-13T14:30:27.500921+0000","last_active":"2024-09-13T14:32:57.953863+0000","last_peered":"2024-09-13T14:32:57.953863+0000","last_clean":"2024-09-13T14:32:57.953863+0000","last_became_active":"2024-09-13T14:30:27.500457+0000","last_became_peered":"2024-09-13T14:30:27.500457+0000","last_unstale":"2024-09-13T14:32:57.953863+0000","last_undegraded":"2024-09-13T14:32:57.953863+0000","last_fullsized":"2024-09-13T14:32:57.953863+0000","mapping_epoch":173,"log_start":"179'3000","ondisk_log_start":"179'3000","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:09.832130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10596,"num_write_kb":499136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":143,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482711+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:32:30.482711+0000","last_peered":"2024-09-13T14:32:30.482711+0000","last_clean":"2024-09-13T14:32:30.482711+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:32:30.482711+0000","last_undegraded":"2024-09-13T14:32:30.482711+0000","last_fullsized":"2024-09-13T14:32:30.482711+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479720+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:32:30.479720+0000","last_peered":"2024-09-13T14:32:30.479720+0000","last_clean":"2024-09-13T14:32:30.479720+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:32:30.479720+0000","last_undegraded":"2024-09-13T14:32:30.479720+0000","last_fullsized":"2024-09-13T14:32:30.479720+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"195'10479","reported_seq":10535,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.560828+0000","last_change":"2024-09-13T14:30:27.501287+0000","last_active":"2024-09-13T14:32:57.560828+0000","last_peered":"2024-09-13T14:32:57.560828+0000","last_clean":"2024-09-13T14:32:57.560828+0000","last_became_active":"2024-09-13T14:30:27.500947+0000","last_became_peered":"2024-09-13T14:30:27.500947+0000","last_unstale":"2024-09-13T14:32:57.560828+0000","last_undegraded":"2024-09-13T14:32:57.560828+0000","last_fullsized":"2024-09-13T14:32:57.560828+0000","mapping_epoch":173,"log_start":"177'1300","ondisk_log_start":"177'1300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9179,"log_dups_size":0,"ondisk_log_size":9179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:44:23.541886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329187328,"num_objects":5023,"num_object_clones":0,"num_object_copies":15069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10479,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491651+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:32:30.491651+0000","last_peered":"2024-09-13T14:32:30.491651+0000","last_clean":"2024-09-13T14:32:30.491651+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:32:30.491651+0000","last_undegraded":"2024-09-13T14:32:30.491651+0000","last_fullsized":"2024-09-13T14:32:30.491651+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546594+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:32:40.546594+0000","last_peered":"2024-09-13T14:32:40.546594+0000","last_clean":"2024-09-13T14:32:40.546594+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:32:40.546594+0000","last_undegraded":"2024-09-13T14:32:40.546594+0000","last_fullsized":"2024-09-13T14:32:40.546594+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"195'10642","reported_seq":10672,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.946239+0000","last_change":"2024-09-13T14:30:27.501830+0000","last_active":"2024-09-13T14:32:57.946239+0000","last_peered":"2024-09-13T14:32:57.946239+0000","last_clean":"2024-09-13T14:32:57.946239+0000","last_became_active":"2024-09-13T14:30:27.501398+0000","last_became_peered":"2024-09-13T14:30:27.501398+0000","last_unstale":"2024-09-13T14:32:57.946239+0000","last_undegraded":"2024-09-13T14:32:57.946239+0000","last_fullsized":"2024-09-13T14:32:57.946239+0000","mapping_epoch":173,"log_start":"180'3300","ondisk_log_start":"180'3300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7342,"log_dups_size":0,"ondisk_log_size":7342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:52.275184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332267520,"num_objects":5070,"num_object_clones":0,"num_object_copies":15210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10642,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":306,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482113+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:32:30.482113+0000","last_peered":"2024-09-13T14:32:30.482113+0000","last_clean":"2024-09-13T14:32:30.482113+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:32:30.482113+0000","last_undegraded":"2024-09-13T14:32:30.482113+0000","last_fullsized":"2024-09-13T14:32:30.482113+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548632+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:32:40.548632+0000","last_peered":"2024-09-13T14:32:40.548632+0000","last_clean":"2024-09-13T14:32:40.548632+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:32:40.548632+0000","last_undegraded":"2024-09-13T14:32:40.548632+0000","last_fullsized":"2024-09-13T14:32:40.548632+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488788+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:32:30.488788+0000","last_peered":"2024-09-13T14:32:30.488788+0000","last_clean":"2024-09-13T14:32:30.488788+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:32:30.488788+0000","last_undegraded":"2024-09-13T14:32:30.488788+0000","last_fullsized":"2024-09-13T14:32:30.488788+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"195'10536","reported_seq":10592,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:56.267502+0000","last_change":"2024-09-13T14:30:27.505193+0000","last_active":"2024-09-13T14:32:56.267502+0000","last_peered":"2024-09-13T14:32:56.267502+0000","last_clean":"2024-09-13T14:32:56.267502+0000","last_became_active":"2024-09-13T14:30:27.504909+0000","last_became_peered":"2024-09-13T14:30:27.504909+0000","last_unstale":"2024-09-13T14:32:56.267502+0000","last_undegraded":"2024-09-13T14:32:56.267502+0000","last_fullsized":"2024-09-13T14:32:56.267502+0000","mapping_epoch":173,"log_start":"177'1100","ondisk_log_start":"177'1100","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9436,"log_dups_size":0,"ondisk_log_size":9436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:40.244032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333447168,"num_objects":5088,"num_object_clones":0,"num_object_copies":15264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10536,"num_write_kb":499968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":53,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490771+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:32:30.490771+0000","last_peered":"2024-09-13T14:32:30.490771+0000","last_clean":"2024-09-13T14:32:30.490771+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:32:30.490771+0000","last_undegraded":"2024-09-13T14:32:30.490771+0000","last_fullsized":"2024-09-13T14:32:30.490771+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.480184+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:32:30.480184+0000","last_peered":"2024-09-13T14:32:30.480184+0000","last_clean":"2024-09-13T14:32:30.480184+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:32:30.480184+0000","last_undegraded":"2024-09-13T14:32:30.480184+0000","last_fullsized":"2024-09-13T14:32:30.480184+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489252+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:32:30.489252+0000","last_peered":"2024-09-13T14:32:30.489252+0000","last_clean":"2024-09-13T14:32:30.489252+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:32:30.489252+0000","last_undegraded":"2024-09-13T14:32:30.489252+0000","last_fullsized":"2024-09-13T14:32:30.489252+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"195'10441","reported_seq":10497,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:54.319299+0000","last_change":"2024-09-13T14:30:27.503642+0000","last_active":"2024-09-13T14:32:54.319299+0000","last_peered":"2024-09-13T14:32:54.319299+0000","last_clean":"2024-09-13T14:32:54.319299+0000","last_became_active":"2024-09-13T14:30:27.503347+0000","last_became_peered":"2024-09-13T14:30:27.503347+0000","last_unstale":"2024-09-13T14:32:54.319299+0000","last_undegraded":"2024-09-13T14:32:54.319299+0000","last_fullsized":"2024-09-13T14:32:54.319299+0000","mapping_epoch":173,"log_start":"180'3900","ondisk_log_start":"180'3900","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6541,"log_dups_size":0,"ondisk_log_size":6541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:40.203221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336396288,"num_objects":5133,"num_object_clones":0,"num_object_copies":15399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10441,"num_write_kb":498368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488419+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:32:30.488419+0000","last_peered":"2024-09-13T14:32:30.488419+0000","last_clean":"2024-09-13T14:32:30.488419+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:32:30.488419+0000","last_undegraded":"2024-09-13T14:32:30.488419+0000","last_fullsized":"2024-09-13T14:32:30.488419+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489889+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:32:30.489889+0000","last_peered":"2024-09-13T14:32:30.489889+0000","last_clean":"2024-09-13T14:32:30.489889+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:32:30.489889+0000","last_undegraded":"2024-09-13T14:32:30.489889+0000","last_fullsized":"2024-09-13T14:32:30.489889+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"195'10550","reported_seq":10581,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.966183+0000","last_change":"2024-09-13T14:30:27.499796+0000","last_active":"2024-09-13T14:32:57.966183+0000","last_peered":"2024-09-13T14:32:57.966183+0000","last_clean":"2024-09-13T14:32:57.966183+0000","last_became_active":"2024-09-13T14:30:27.499475+0000","last_became_peered":"2024-09-13T14:30:27.499475+0000","last_unstale":"2024-09-13T14:32:57.966183+0000","last_undegraded":"2024-09-13T14:32:57.966183+0000","last_fullsized":"2024-09-13T14:32:57.966183+0000","mapping_epoch":173,"log_start":"177'500","ondisk_log_start":"177'500","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10050,"log_dups_size":0,"ondisk_log_size":10050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:07.838538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328794136,"num_objects":5018,"num_object_clones":0,"num_object_copies":15054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5018,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10550,"num_write_kb":498113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490978+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:32:30.490978+0000","last_peered":"2024-09-13T14:32:30.490978+0000","last_clean":"2024-09-13T14:32:30.490978+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:32:30.490978+0000","last_undegraded":"2024-09-13T14:32:30.490978+0000","last_fullsized":"2024-09-13T14:32:30.490978+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":259,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.549021+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:32:40.549021+0000","last_peered":"2024-09-13T14:32:40.549021+0000","last_clean":"2024-09-13T14:32:40.549021+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:32:40.549021+0000","last_undegraded":"2024-09-13T14:32:40.549021+0000","last_fullsized":"2024-09-13T14:32:40.549021+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533516+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:32:40.533516+0000","last_peered":"2024-09-13T14:32:40.533516+0000","last_clean":"2024-09-13T14:32:40.533516+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:32:40.533516+0000","last_undegraded":"2024-09-13T14:32:40.533516+0000","last_fullsized":"2024-09-13T14:32:40.533516+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"195'10787","reported_seq":10817,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.975099+0000","last_change":"2024-09-13T14:30:27.505472+0000","last_active":"2024-09-13T14:32:57.975099+0000","last_peered":"2024-09-13T14:32:57.975099+0000","last_clean":"2024-09-13T14:32:57.975099+0000","last_became_active":"2024-09-13T14:30:27.505098+0000","last_became_peered":"2024-09-13T14:30:27.505098+0000","last_unstale":"2024-09-13T14:32:57.975099+0000","last_undegraded":"2024-09-13T14:32:57.975099+0000","last_fullsized":"2024-09-13T14:32:57.975099+0000","mapping_epoch":173,"log_start":"177'700","ondisk_log_start":"177'700","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10087,"log_dups_size":0,"ondisk_log_size":10087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:45.218708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329449472,"num_objects":5027,"num_object_clones":0,"num_object_copies":15081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10787,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":297,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482338+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:32:30.482338+0000","last_peered":"2024-09-13T14:32:30.482338+0000","last_clean":"2024-09-13T14:32:30.482338+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:32:30.482338+0000","last_undegraded":"2024-09-13T14:32:30.482338+0000","last_fullsized":"2024-09-13T14:32:30.482338+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":760,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490944+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:32:30.490944+0000","last_peered":"2024-09-13T14:32:30.490944+0000","last_clean":"2024-09-13T14:32:30.490944+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:32:30.490944+0000","last_undegraded":"2024-09-13T14:32:30.490944+0000","last_fullsized":"2024-09-13T14:32:30.490944+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":212,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548097+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:32:40.548097+0000","last_peered":"2024-09-13T14:32:40.548097+0000","last_clean":"2024-09-13T14:32:40.548097+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:32:40.548097+0000","last_undegraded":"2024-09-13T14:32:40.548097+0000","last_fullsized":"2024-09-13T14:32:40.548097+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":141,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481839+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:32:30.481839+0000","last_peered":"2024-09-13T14:32:30.481839+0000","last_clean":"2024-09-13T14:32:30.481839+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:32:30.481839+0000","last_undegraded":"2024-09-13T14:32:30.481839+0000","last_fullsized":"2024-09-13T14:32:30.481839+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"195'10682","reported_seq":10738,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.341691+0000","last_change":"2024-09-13T14:30:27.505328+0000","last_active":"2024-09-13T14:32:57.341691+0000","last_peered":"2024-09-13T14:32:57.341691+0000","last_clean":"2024-09-13T14:32:57.341691+0000","last_became_active":"2024-09-13T14:30:27.505015+0000","last_became_peered":"2024-09-13T14:30:27.505015+0000","last_unstale":"2024-09-13T14:32:57.341691+0000","last_undegraded":"2024-09-13T14:32:57.341691+0000","last_fullsized":"2024-09-13T14:32:57.341691+0000","mapping_epoch":173,"log_start":"179'2700","ondisk_log_start":"179'2700","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7982,"log_dups_size":0,"ondisk_log_size":7982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:42.013686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330694656,"num_objects":5046,"num_object_clones":0,"num_object_copies":15138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10682,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":89,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534352+0000","last_change":"2024-09-13T14:30:39.822827+0000","last_active":"2024-09-13T14:32:40.534352+0000","last_peered":"2024-09-13T14:32:40.534352+0000","last_clean":"2024-09-13T14:32:40.534352+0000","last_became_active":"2024-09-13T14:30:11.437578+0000","last_became_peered":"2024-09-13T14:30:11.437578+0000","last_unstale":"2024-09-13T14:32:40.534352+0000","last_undegraded":"2024-09-13T14:32:40.534352+0000","last_fullsized":"2024-09-13T14:32:40.534352+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.822747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.822747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:44.769254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":568,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482196+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:32:30.482196+0000","last_peered":"2024-09-13T14:32:30.482196+0000","last_clean":"2024-09-13T14:32:30.482196+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:32:30.482196+0000","last_undegraded":"2024-09-13T14:32:30.482196+0000","last_fullsized":"2024-09-13T14:32:30.482196+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":254,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546629+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:32:40.546629+0000","last_peered":"2024-09-13T14:32:40.546629+0000","last_clean":"2024-09-13T14:32:40.546629+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:32:40.546629+0000","last_undegraded":"2024-09-13T14:32:40.546629+0000","last_fullsized":"2024-09-13T14:32:40.546629+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545947+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:32:40.545947+0000","last_peered":"2024-09-13T14:32:40.545947+0000","last_clean":"2024-09-13T14:32:40.545947+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:32:40.545947+0000","last_undegraded":"2024-09-13T14:32:40.545947+0000","last_fullsized":"2024-09-13T14:32:40.545947+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"195'10441","reported_seq":10471,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.995203+0000","last_change":"2024-09-13T14:30:27.500479+0000","last_active":"2024-09-13T14:32:57.995203+0000","last_peered":"2024-09-13T14:32:57.995203+0000","last_clean":"2024-09-13T14:32:57.995203+0000","last_became_active":"2024-09-13T14:30:27.500110+0000","last_became_peered":"2024-09-13T14:30:27.500110+0000","last_unstale":"2024-09-13T14:32:57.995203+0000","last_undegraded":"2024-09-13T14:32:57.995203+0000","last_fullsized":"2024-09-13T14:32:57.995203+0000","mapping_epoch":173,"log_start":"179'2900","ondisk_log_start":"179'2900","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:10.174744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326303744,"num_objects":4979,"num_object_clones":0,"num_object_copies":14937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10441,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":306,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.481360+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:32:30.481360+0000","last_peered":"2024-09-13T14:32:30.481360+0000","last_clean":"2024-09-13T14:32:30.481360+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:32:30.481360+0000","last_undegraded":"2024-09-13T14:32:30.481360+0000","last_fullsized":"2024-09-13T14:32:30.481360+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":683,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.543413+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:32:40.543413+0000","last_peered":"2024-09-13T14:32:40.543413+0000","last_clean":"2024-09-13T14:32:40.543413+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:32:40.543413+0000","last_undegraded":"2024-09-13T14:32:40.543413+0000","last_fullsized":"2024-09-13T14:32:40.543413+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":245,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533763+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:32:40.533763+0000","last_peered":"2024-09-13T14:32:40.533763+0000","last_clean":"2024-09-13T14:32:40.533763+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:32:40.533763+0000","last_undegraded":"2024-09-13T14:32:40.533763+0000","last_fullsized":"2024-09-13T14:32:40.533763+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538164+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:32:40.538164+0000","last_peered":"2024-09-13T14:32:40.538164+0000","last_clean":"2024-09-13T14:32:40.538164+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:32:40.538164+0000","last_undegraded":"2024-09-13T14:32:40.538164+0000","last_fullsized":"2024-09-13T14:32:40.538164+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"195'10363","reported_seq":10419,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:54.332831+0000","last_change":"2024-09-13T14:30:27.501260+0000","last_active":"2024-09-13T14:32:54.332831+0000","last_peered":"2024-09-13T14:32:54.332831+0000","last_clean":"2024-09-13T14:32:54.332831+0000","last_became_active":"2024-09-13T14:30:27.500950+0000","last_became_peered":"2024-09-13T14:30:27.500950+0000","last_unstale":"2024-09-13T14:32:54.332831+0000","last_undegraded":"2024-09-13T14:32:54.332831+0000","last_fullsized":"2024-09-13T14:32:54.332831+0000","mapping_epoch":173,"log_start":"180'3800","ondisk_log_start":"180'3800","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:54.878021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338755584,"num_objects":5169,"num_object_clones":0,"num_object_copies":15507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10363,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.490923+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:32:30.490923+0000","last_peered":"2024-09-13T14:32:30.490923+0000","last_clean":"2024-09-13T14:32:30.490923+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:32:30.490923+0000","last_undegraded":"2024-09-13T14:32:30.490923+0000","last_fullsized":"2024-09-13T14:32:30.490923+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":809,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491109+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:32:30.491109+0000","last_peered":"2024-09-13T14:32:30.491109+0000","last_clean":"2024-09-13T14:32:30.491109+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:32:30.491109+0000","last_undegraded":"2024-09-13T14:32:30.491109+0000","last_fullsized":"2024-09-13T14:32:30.491109+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488977+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:32:30.488977+0000","last_peered":"2024-09-13T14:32:30.488977+0000","last_clean":"2024-09-13T14:32:30.488977+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:32:30.488977+0000","last_undegraded":"2024-09-13T14:32:30.488977+0000","last_fullsized":"2024-09-13T14:32:30.488977+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534022+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:32:40.534022+0000","last_peered":"2024-09-13T14:32:40.534022+0000","last_clean":"2024-09-13T14:32:40.534022+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:32:40.534022+0000","last_undegraded":"2024-09-13T14:32:40.534022+0000","last_fullsized":"2024-09-13T14:32:40.534022+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"195'10256","reported_seq":10309,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.562496+0000","last_change":"2024-09-13T14:30:27.502045+0000","last_active":"2024-09-13T14:32:57.562496+0000","last_peered":"2024-09-13T14:32:57.562496+0000","last_clean":"2024-09-13T14:32:57.562496+0000","last_became_active":"2024-09-13T14:30:27.501711+0000","last_became_peered":"2024-09-13T14:30:27.501711+0000","last_unstale":"2024-09-13T14:32:57.562496+0000","last_undegraded":"2024-09-13T14:32:57.562496+0000","last_fullsized":"2024-09-13T14:32:57.562496+0000","mapping_epoch":173,"log_start":"177'1100","ondisk_log_start":"177'1100","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9156,"log_dups_size":0,"ondisk_log_size":9156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:21.043630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322306048,"num_objects":4918,"num_object_clones":0,"num_object_copies":14754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10256,"num_write_kb":485568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":194,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479838+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:32:30.479838+0000","last_peered":"2024-09-13T14:32:30.479838+0000","last_clean":"2024-09-13T14:32:30.479838+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:32:30.479838+0000","last_undegraded":"2024-09-13T14:32:30.479838+0000","last_fullsized":"2024-09-13T14:32:30.479838+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491260+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:32:30.491260+0000","last_peered":"2024-09-13T14:32:30.491260+0000","last_clean":"2024-09-13T14:32:30.491260+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:32:30.491260+0000","last_undegraded":"2024-09-13T14:32:30.491260+0000","last_fullsized":"2024-09-13T14:32:30.491260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":854,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534173+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:32:40.534173+0000","last_peered":"2024-09-13T14:32:40.534173+0000","last_clean":"2024-09-13T14:32:40.534173+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:32:40.534173+0000","last_undegraded":"2024-09-13T14:32:40.534173+0000","last_fullsized":"2024-09-13T14:32:40.534173+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":233,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533309+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:32:40.533309+0000","last_peered":"2024-09-13T14:32:40.533309+0000","last_clean":"2024-09-13T14:32:40.533309+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:32:40.533309+0000","last_undegraded":"2024-09-13T14:32:40.533309+0000","last_fullsized":"2024-09-13T14:32:40.533309+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"195'10522","reported_seq":10578,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.447817+0000","last_change":"2024-09-13T14:30:27.500989+0000","last_active":"2024-09-13T14:32:57.447817+0000","last_peered":"2024-09-13T14:32:57.447817+0000","last_clean":"2024-09-13T14:32:57.447817+0000","last_became_active":"2024-09-13T14:30:27.500671+0000","last_became_peered":"2024-09-13T14:30:27.500671+0000","last_unstale":"2024-09-13T14:32:57.447817+0000","last_undegraded":"2024-09-13T14:32:57.447817+0000","last_fullsized":"2024-09-13T14:32:57.447817+0000","mapping_epoch":173,"log_start":"179'2600","ondisk_log_start":"179'2600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7922,"log_dups_size":0,"ondisk_log_size":7922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:59.179207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342228992,"num_objects":5222,"num_object_clones":0,"num_object_copies":15666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10522,"num_write_kb":503808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":194,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479685+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:32:30.479685+0000","last_peered":"2024-09-13T14:32:30.479685+0000","last_clean":"2024-09-13T14:32:30.479685+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:32:30.479685+0000","last_undegraded":"2024-09-13T14:32:30.479685+0000","last_fullsized":"2024-09-13T14:32:30.479685+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.534094+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:32:40.534094+0000","last_peered":"2024-09-13T14:32:40.534094+0000","last_clean":"2024-09-13T14:32:40.534094+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:32:40.534094+0000","last_undegraded":"2024-09-13T14:32:40.534094+0000","last_fullsized":"2024-09-13T14:32:40.534094+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":505,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482720+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:32:30.482720+0000","last_peered":"2024-09-13T14:32:30.482720+0000","last_clean":"2024-09-13T14:32:30.482720+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:32:30.482720+0000","last_undegraded":"2024-09-13T14:32:30.482720+0000","last_fullsized":"2024-09-13T14:32:30.482720+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546489+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:32:40.546489+0000","last_peered":"2024-09-13T14:32:40.546489+0000","last_clean":"2024-09-13T14:32:40.546489+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:32:40.546489+0000","last_undegraded":"2024-09-13T14:32:40.546489+0000","last_fullsized":"2024-09-13T14:32:40.546489+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"195'10678","reported_seq":10708,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.969618+0000","last_change":"2024-09-13T14:30:27.502612+0000","last_active":"2024-09-13T14:32:57.969618+0000","last_peered":"2024-09-13T14:32:57.969618+0000","last_clean":"2024-09-13T14:32:57.969618+0000","last_became_active":"2024-09-13T14:30:27.501659+0000","last_became_peered":"2024-09-13T14:30:27.501659+0000","last_unstale":"2024-09-13T14:32:57.969618+0000","last_undegraded":"2024-09-13T14:32:57.969618+0000","last_fullsized":"2024-09-13T14:32:57.969618+0000","mapping_epoch":173,"log_start":"180'3300","ondisk_log_start":"180'3300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7378,"log_dups_size":0,"ondisk_log_size":7378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:47.542650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330039296,"num_objects":5036,"num_object_clones":0,"num_object_copies":15108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10678,"num_write_kb":502848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":46,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489054+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:32:30.489054+0000","last_peered":"2024-09-13T14:32:30.489054+0000","last_clean":"2024-09-13T14:32:30.489054+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:32:30.489054+0000","last_undegraded":"2024-09-13T14:32:30.489054+0000","last_fullsized":"2024-09-13T14:32:30.489054+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":256,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.552278+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:32:40.552278+0000","last_peered":"2024-09-13T14:32:40.552278+0000","last_clean":"2024-09-13T14:32:40.552278+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:32:40.552278+0000","last_undegraded":"2024-09-13T14:32:40.552278+0000","last_fullsized":"2024-09-13T14:32:40.552278+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":588,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533469+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:32:40.533469+0000","last_peered":"2024-09-13T14:32:40.533469+0000","last_clean":"2024-09-13T14:32:40.533469+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:32:40.533469+0000","last_undegraded":"2024-09-13T14:32:40.533469+0000","last_fullsized":"2024-09-13T14:32:40.533469+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":253,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548468+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:32:40.548468+0000","last_peered":"2024-09-13T14:32:40.548468+0000","last_clean":"2024-09-13T14:32:40.548468+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:32:40.548468+0000","last_undegraded":"2024-09-13T14:32:40.548468+0000","last_fullsized":"2024-09-13T14:32:40.548468+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"195'10678","reported_seq":10708,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:57.962401+0000","last_change":"2024-09-13T14:30:27.501009+0000","last_active":"2024-09-13T14:32:57.962401+0000","last_peered":"2024-09-13T14:32:57.962401+0000","last_clean":"2024-09-13T14:32:57.962401+0000","last_became_active":"2024-09-13T14:30:27.500481+0000","last_became_peered":"2024-09-13T14:30:27.500481+0000","last_unstale":"2024-09-13T14:32:57.962401+0000","last_undegraded":"2024-09-13T14:32:57.962401+0000","last_fullsized":"2024-09-13T14:32:57.962401+0000","mapping_epoch":173,"log_start":"179'3100","ondisk_log_start":"179'3100","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:30.536488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10678,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":94,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.551540+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:32:40.551540+0000","last_peered":"2024-09-13T14:32:40.551540+0000","last_clean":"2024-09-13T14:32:40.551540+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:32:40.551540+0000","last_undegraded":"2024-09-13T14:32:40.551540+0000","last_fullsized":"2024-09-13T14:32:40.551540+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":258,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545696+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:32:40.545696+0000","last_peered":"2024-09-13T14:32:40.545696+0000","last_clean":"2024-09-13T14:32:40.545696+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:32:40.545696+0000","last_undegraded":"2024-09-13T14:32:40.545696+0000","last_fullsized":"2024-09-13T14:32:40.545696+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":613,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.480031+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:32:30.480031+0000","last_peered":"2024-09-13T14:32:30.480031+0000","last_clean":"2024-09-13T14:32:30.480031+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:32:30.480031+0000","last_undegraded":"2024-09-13T14:32:30.480031+0000","last_fullsized":"2024-09-13T14:32:30.480031+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":142,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488245+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:32:30.488245+0000","last_peered":"2024-09-13T14:32:30.488245+0000","last_clean":"2024-09-13T14:32:30.488245+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:32:30.488245+0000","last_undegraded":"2024-09-13T14:32:30.488245+0000","last_fullsized":"2024-09-13T14:32:30.488245+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488859+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:32:30.488859+0000","last_peered":"2024-09-13T14:32:30.488859+0000","last_clean":"2024-09-13T14:32:30.488859+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:32:30.488859+0000","last_undegraded":"2024-09-13T14:32:30.488859+0000","last_fullsized":"2024-09-13T14:32:30.488859+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":165,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488876+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:32:30.488876+0000","last_peered":"2024-09-13T14:32:30.488876+0000","last_clean":"2024-09-13T14:32:30.488876+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:32:30.488876+0000","last_undegraded":"2024-09-13T14:32:30.488876+0000","last_fullsized":"2024-09-13T14:32:30.488876+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":244,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533148+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:32:40.533148+0000","last_peered":"2024-09-13T14:32:40.533148+0000","last_clean":"2024-09-13T14:32:40.533148+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:32:40.533148+0000","last_undegraded":"2024-09-13T14:32:40.533148+0000","last_fullsized":"2024-09-13T14:32:40.533148+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":162,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.482559+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:32:30.482559+0000","last_peered":"2024-09-13T14:32:30.482559+0000","last_clean":"2024-09-13T14:32:30.482559+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:32:30.482559+0000","last_undegraded":"2024-09-13T14:32:30.482559+0000","last_fullsized":"2024-09-13T14:32:30.482559+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":159,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491106+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:32:30.491106+0000","last_peered":"2024-09-13T14:32:30.491106+0000","last_clean":"2024-09-13T14:32:30.491106+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:32:30.491106+0000","last_undegraded":"2024-09-13T14:32:30.491106+0000","last_fullsized":"2024-09-13T14:32:30.491106+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":212,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548175+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:32:40.548175+0000","last_peered":"2024-09-13T14:32:40.548175+0000","last_clean":"2024-09-13T14:32:40.548175+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:32:40.548175+0000","last_undegraded":"2024-09-13T14:32:40.548175+0000","last_fullsized":"2024-09-13T14:32:40.548175+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":253,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546128+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:32:40.546128+0000","last_peered":"2024-09-13T14:32:40.546128+0000","last_clean":"2024-09-13T14:32:40.546128+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:32:40.546128+0000","last_undegraded":"2024-09-13T14:32:40.546128+0000","last_fullsized":"2024-09-13T14:32:40.546128+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548362+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:32:40.548362+0000","last_peered":"2024-09-13T14:32:40.548362+0000","last_clean":"2024-09-13T14:32:40.548362+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:32:40.548362+0000","last_undegraded":"2024-09-13T14:32:40.548362+0000","last_fullsized":"2024-09-13T14:32:40.548362+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.479916+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:32:30.479916+0000","last_peered":"2024-09-13T14:32:30.479916+0000","last_clean":"2024-09-13T14:32:30.479916+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:32:30.479916+0000","last_undegraded":"2024-09-13T14:32:30.479916+0000","last_fullsized":"2024-09-13T14:32:30.479916+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":275,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548768+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:32:40.548768+0000","last_peered":"2024-09-13T14:32:40.548768+0000","last_clean":"2024-09-13T14:32:40.548768+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:32:40.548768+0000","last_undegraded":"2024-09-13T14:32:40.548768+0000","last_fullsized":"2024-09-13T14:32:40.548768+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.545864+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:32:40.545864+0000","last_peered":"2024-09-13T14:32:40.545864+0000","last_clean":"2024-09-13T14:32:40.545864+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:32:40.545864+0000","last_undegraded":"2024-09-13T14:32:40.545864+0000","last_fullsized":"2024-09-13T14:32:40.545864+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":245,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.533831+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:32:40.533831+0000","last_peered":"2024-09-13T14:32:40.533831+0000","last_clean":"2024-09-13T14:32:40.533831+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:32:40.533831+0000","last_undegraded":"2024-09-13T14:32:40.533831+0000","last_fullsized":"2024-09-13T14:32:40.533831+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":257,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548850+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:32:40.548850+0000","last_peered":"2024-09-13T14:32:40.548850+0000","last_clean":"2024-09-13T14:32:40.548850+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:32:40.548850+0000","last_undegraded":"2024-09-13T14:32:40.548850+0000","last_fullsized":"2024-09-13T14:32:40.548850+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":178,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489279+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:32:30.489279+0000","last_peered":"2024-09-13T14:32:30.489279+0000","last_clean":"2024-09-13T14:32:30.489279+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:32:30.489279+0000","last_undegraded":"2024-09-13T14:32:30.489279+0000","last_fullsized":"2024-09-13T14:32:30.489279+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.537703+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:32:40.537703+0000","last_peered":"2024-09-13T14:32:40.537703+0000","last_clean":"2024-09-13T14:32:40.537703+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:32:40.537703+0000","last_undegraded":"2024-09-13T14:32:40.537703+0000","last_fullsized":"2024-09-13T14:32:40.537703+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":246,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546395+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:32:40.546395+0000","last_peered":"2024-09-13T14:32:40.546395+0000","last_clean":"2024-09-13T14:32:40.546395+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:32:40.546395+0000","last_undegraded":"2024-09-13T14:32:40.546395+0000","last_fullsized":"2024-09-13T14:32:40.546395+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488788+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:32:30.488788+0000","last_peered":"2024-09-13T14:32:30.488788+0000","last_clean":"2024-09-13T14:32:30.488788+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:32:30.488788+0000","last_undegraded":"2024-09-13T14:32:30.488788+0000","last_fullsized":"2024-09-13T14:32:30.488788+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491534+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:32:30.491534+0000","last_peered":"2024-09-13T14:32:30.491534+0000","last_clean":"2024-09-13T14:32:30.491534+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:32:30.491534+0000","last_undegraded":"2024-09-13T14:32:30.491534+0000","last_fullsized":"2024-09-13T14:32:30.491534+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.489198+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:32:30.489198+0000","last_peered":"2024-09-13T14:32:30.489198+0000","last_clean":"2024-09-13T14:32:30.489198+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:32:30.489198+0000","last_undegraded":"2024-09-13T14:32:30.489198+0000","last_fullsized":"2024-09-13T14:32:30.489198+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":144,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.491242+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:32:30.491242+0000","last_peered":"2024-09-13T14:32:30.491242+0000","last_clean":"2024-09-13T14:32:30.491242+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:32:30.491242+0000","last_undegraded":"2024-09-13T14:32:30.491242+0000","last_fullsized":"2024-09-13T14:32:30.491242+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":278,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.548471+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:32:40.548471+0000","last_peered":"2024-09-13T14:32:40.548471+0000","last_clean":"2024-09-13T14:32:40.548471+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:32:40.548471+0000","last_undegraded":"2024-09-13T14:32:40.548471+0000","last_fullsized":"2024-09-13T14:32:40.548471+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":264,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.551013+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:32:40.551013+0000","last_peered":"2024-09-13T14:32:40.551013+0000","last_clean":"2024-09-13T14:32:40.551013+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:32:40.551013+0000","last_undegraded":"2024-09-13T14:32:40.551013+0000","last_fullsized":"2024-09-13T14:32:40.551013+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2024-09-13T14:32:30.488096+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:32:30.488096+0000","last_peered":"2024-09-13T14:32:30.488096+0000","last_clean":"2024-09-13T14:32:30.488096+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:32:30.488096+0000","last_undegraded":"2024-09-13T14:32:30.488096+0000","last_fullsized":"2024-09-13T14:32:30.488096+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":258,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.546044+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:32:40.546044+0000","last_peered":"2024-09-13T14:32:40.546044+0000","last_clean":"2024-09-13T14:32:40.546044+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:32:40.546044+0000","last_undegraded":"2024-09-13T14:32:40.546044+0000","last_fullsized":"2024-09-13T14:32:40.546044+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5302845464,"num_objects":80916,"num_object_clones":0,"num_object_copies":242748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168304,"num_write_kb":7974977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15942758400,"data_stored":15942746184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131004,"ondisk_log_size":131004,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397762,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2242608,"kb_used_data":1932776,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91502544,"statfs":{"total":95995035648,"available":93698605056,"internally_reserved":0,"allocated":1979162624,"data_stored":1978140377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593990,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1927264,"kb_used_data":1621592,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91817888,"statfs":{"total":95995035648,"available":94021517312,"internally_reserved":0,"allocated":1660510208,"data_stored":1659503651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790220,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1908576,"kb_used_data":1607064,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91836576,"statfs":{"total":95995035648,"available":94040653824,"internally_reserved":0,"allocated":1645633536,"data_stored":1644623310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986448,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1921900,"kb_used_data":1612068,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91823252,"statfs":{"total":95995035648,"available":94027010048,"internally_reserved":0,"allocated":1650757632,"data_stored":1649739411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697005,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3667868,"kb_used_data":3335956,"kb_used_omap":4,"kb_used_meta":331899,"kb_avail":90077284,"statfs":{"total":95995035648,"available":92239138816,"internally_reserved":0,"allocated":3416018944,"data_stored":3415012863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":339864651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860532,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2958368,"kb_used_data":2638616,"kb_used_omap":8,"kb_used_meta":319543,"kb_avail":90786784,"statfs":{"total":95995035648,"available":92965666816,"internally_reserved":0,"allocated":2701942784,"data_stored":2700933129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":327212876},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024059,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1943004,"kb_used_data":1646868,"kb_used_omap":7,"kb_used_meta":295992,"kb_avail":91802148,"statfs":{"total":95995035648,"available":94005399552,"internally_reserved":0,"allocated":1686392832,"data_stored":1685386152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":303096247},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187594,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579100,"kb_used_data":1288596,"kb_used_omap":7,"kb_used_meta":290488,"kb_avail":92166052,"statfs":{"total":95995035648,"available":94378037248,"internally_reserved":0,"allocated":1319522304,"data_stored":1318515898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":297460043},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1305743360,"data_stored":1305739288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1665536000,"data_stored":1665531928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2688024576,"data_stored":2688024576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3381526528,"data_stored":3381526528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1643970560,"data_stored":1643970560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1635127296,"data_stored":1635123224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1653014528,"data_stored":1653014528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1969815552,"data_stored":1969815552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:33:01.445 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:33:01.446 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:33:01.446 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-09-13T14:33:01.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:33:01.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:01.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:01.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:01.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:01.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:01 smithi017 ceph-mon[103001]: from='client.45407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:01 smithi017 ceph-mon[103001]: pgmap v561: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:01 smithi017 ceph-mon[107556]: from='client.45407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:01 smithi017 ceph-mon[107556]: pgmap v561: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:01.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:01 smithi189 ceph-mon[90992]: from='client.45407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:01 smithi189 ceph-mon[90992]: pgmap v561: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:01.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:01.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:01.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:01.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:02.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:02.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:02.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:02.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:02.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:02.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:02.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:02.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:02.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:02.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:02.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:02.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:33:02.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:02.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:02.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:02.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:02.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:03.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:03.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:03.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:03.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:03.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:03.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:03.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:33:03.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:03.486 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:03.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:03.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:03 smithi017 ceph-mon[103001]: pgmap v562: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-09-13T14:33:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:03 smithi017 ceph-mon[107556]: pgmap v562: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-09-13T14:33:03.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:03 smithi189 ceph-mon[90992]: pgmap v562: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-09-13T14:33:03.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:03.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:03.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:03.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:03.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:04.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:04.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:04.095 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:04.099 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:04.179 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:04.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:33:04.182 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:33:04.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:04.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:33:04.219 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:33:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.222 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:04.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:04.257 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:33:04.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:33:04.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:33:04.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:33:04.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:04.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:33:04.326 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:33:04.358 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:33:04.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:33:04.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:04.388 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:33:04.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:33:04.425 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:33:04.425 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:33:04.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:04.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:04.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:04.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/598446896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/598446896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/598446896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:04.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:04.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:04.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:04.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:04.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:04.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:04.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:05.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:05.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:05.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:05.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:05.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:05.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:05.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:05.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:05.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:05 smithi017 ceph-mon[103001]: pgmap v563: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:05.537 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:05 smithi017 ceph-mon[107556]: pgmap v563: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:05.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:05.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:05.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:05 smithi189 ceph-mon[90992]: pgmap v563: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:05.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:05.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:05.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:05.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:06.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:06.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:06.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:06.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:06.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:06.294 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:06.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:06.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.428 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:06.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:06.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:06.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:06.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:06.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:06.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:06.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:06.963 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:06.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:07.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:07.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:07.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:07.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:07.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:07.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:07.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:07.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:07.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:07.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:07] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-13T14:33:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:07 smithi017 ceph-mon[103001]: pgmap v564: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:07 smithi017 ceph-mon[107556]: pgmap v564: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:07.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:07 smithi189 ceph-mon[90992]: pgmap v564: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-09-13T14:33:07.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:07.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:07.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:07.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:07.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:08.028 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:08.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:08.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:08.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:08.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.298 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:08.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:08.399 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:08.425 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:08.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:08.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:08.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:08.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:08.681 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:08.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:08.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:08.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:08.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:08.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:09.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:09.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:09.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:09.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178588 6.6 6.9eeea436 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315720+0000", 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0021980260000000001, 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022251300000000001, 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.321 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178588 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315720+0000", 2024-09-13T14:33:09.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315720+0000", 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315721+0000", 2024-09-13T14:33:09.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.376e-06 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315726+0000", 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.429e-06 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315744+0000", 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7193999999999998e-05 2024-09-13T14:33:09.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315751+0000", 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9030000000000004e-06 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178584 6.6 e196/173)", 2024-09-13T14:33:09.325 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315774+0000", 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002143112, 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022313459999999999, 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315774+0000", 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315774+0000", 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315775+0000", 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8999999999999997e-07 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315794+0000", 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8688000000000001e-05 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315806+0000", 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2225999999999999e-05 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315812+0000", 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.643e-06 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:178587 6.0 e196/173 6:04bedeb0:::benchmark_data_smithi017_185083_object53975:head v 196'10826, mlcod=196'10823)", 2024-09-13T14:33:09.329 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315851+0000", 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0020663460000000002, 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021965330000000001, 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315851+0000", 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315851+0000", 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315852+0000", 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0400000000000002e-07 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315856+0000", 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4399999999999998e-06 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315863+0000", 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.024e-06 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315868+0000", 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.583e-06 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316370+0000", 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050242199999999998 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316389+0000", 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9072e-05 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178590 6.c 6.ae952dcc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.316377+0000", 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0015406269999999999, 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001718324, 2024-09-13T14:33:09.334 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178590 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316377+0000", 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316377+0000", 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316378+0000", 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0699999999999996e-07 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316382+0000", 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5480000000000003e-06 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316392+0000", 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4809999999999995e-06 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316398+0000", 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5889999999999999e-06 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:178558 6.4 e196/173 6:28a9690b:::benchmark_data_smithi017_185083_object53946:head v 196'11367, mlcod=196'11364)", 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.280994+0000", 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.036923932999999999, 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037145969000000001, 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.280994+0000", 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.280994+0000", 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.280995+0000", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.15e-06 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.281002+0000", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6420000000000007e-06 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.281017+0000", 2024-09-13T14:33:09.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5007e-05 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.281025+0000", 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5270000000000001e-06 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.289427+0000", 2024-09-13T14:33:09.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084015860000000008 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.289458+0000", 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1238000000000001e-05 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.344 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178565 6.c 6:3ffae7b5:::benchmark_data_smithi017_185083_object53953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.284268+0000", 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.033650038, 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033924573999999999, 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178565 2024-09-13T14:33:09.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284268+0000", 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284268+0000", 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284269+0000", 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.841e-06 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284275+0000", 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7479999999999999e-06 2024-09-13T14:33:09.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284292+0000", 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6782e-05 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284300+0000", 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0970000000000006e-06 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.299162+0000", 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01486211 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.305219+0000", 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060564700000000004 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:33:09.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.305334+0000", 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114973 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:178559 6.4 e196/173 6:2e46ade4:::benchmark_data_smithi017_185083_object53947:head v 196'11368, mlcod=196'11364)", 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.284904+0000", 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.033013955999999997, 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033350537, 2024-09-13T14:33:09.350 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284904+0000", 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284904+0000", 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284904+0000", 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0299999999999998e-07 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284910+0000", 2024-09-13T14:33:09.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9490000000000001e-06 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284924+0000", 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3601000000000001e-05 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.284930+0000", 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2870000000000001e-06 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.294454+0000", 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095235149999999998 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.294508+0000", 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4453000000000002e-05 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178567 6.c 6:3b8722d9:::benchmark_data_smithi017_185083_object53955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.285171+0000", 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.032746996, 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033136591, 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.355 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178567 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285171+0000", 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285171+0000", 2024-09-13T14:33:09.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285172+0000", 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.206e-06 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285189+0000", 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6926e-05 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285204+0000", 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5434000000000002e-05 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.285212+0000", 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.52e-06 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.309808+0000", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024595881 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.309938+0000", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013061600000000001 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.310324+0000", 2024-09-13T14:33:09.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038545499999999999 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:178562 6.4 e196/173 6:2300d110:::benchmark_data_smithi017_185083_object53950:head v 196'11369, mlcod=196'11364)", 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.290934+0000", 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.026983638000000001, 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027441342000000001, 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.360 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290934+0000", 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290934+0000", 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290934+0000", 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0200000000000002e-07 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290938+0000", 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7120000000000002e-06 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290946+0000", 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7170000000000008e-06 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.290951+0000", 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1200000000000001e-06 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.299086+0000", 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081351789999999993 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.299176+0000", 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9788000000000002e-05 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35349.0:178571 6.4 e196/173 6:2fc8e3ef:::benchmark_data_smithi017_185083_object53959:head v 196'11370, mlcod=196'11364)", 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.300447+0000", 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.017470627999999998, 2024-09-13T14:33:09.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017981990999999999, 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.300447+0000", 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.300447+0000", 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.300448+0000", 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8400000000000003e-07 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.300453+0000", 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5620000000000003e-06 2024-09-13T14:33:09.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.305784+0000", 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053305419999999997 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.305791+0000", 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2250000000000003e-06 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.309557+0000", 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037662020000000002 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.309574+0000", 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7142999999999998e-05 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.368 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178582 6.b 6:d948e629:::benchmark_data_smithi017_185083_object53970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.307067+0000", 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.010850077, 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011418452000000001, 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178582 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307067+0000", 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307067+0000", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307069+0000", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8080000000000001e-06 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307074+0000", 2024-09-13T14:33:09.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8609999999999997e-06 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307087+0000", 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2548000000000001e-05 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.307094+0000", 2024-09-13T14:33:09.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1300000000000003e-06 2024-09-13T14:33:09.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311635+0000", 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045412830000000001 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311742+0000", 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010666 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311998+0000", 2024-09-13T14:33:09.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025579699999999999 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.316234+0000", 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042367430000000003 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.377 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178565 6.c e196/173)", 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.311261+0000", 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0066561279999999999, 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072925689999999996, 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311261+0000", 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311261+0000", 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311262+0000", 2024-09-13T14:33:09.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1699999999999997e-07 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311266+0000", 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5949999999999999e-06 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311280+0000", 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4514e-05 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311288+0000", 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7239999999999992e-06 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178565 6.c e196/173)", 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.311318+0000", 2024-09-13T14:33:09.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0065992760000000003, 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072745190000000001, 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311318+0000", 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311318+0000", 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311319+0000", 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.39e-07 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311321+0000", 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3130000000000001e-06 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311333+0000", 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1600000000000001e-05 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311338+0000", 2024-09-13T14:33:09.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1749999999999996e-06 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178584 6.6 6:67f0529b:::benchmark_data_smithi017_185083_object53972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.311415+0000", 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0065025029999999998, 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072202439999999998, 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.385 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178584 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311415+0000", 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311415+0000", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311416+0000", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1200000000000001e-06 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311420+0000", 2024-09-13T14:33:09.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7060000000000002e-06 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311431+0000", 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1041e-05 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311438+0000", 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3610000000000003e-06 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311498+0000", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9747999999999998e-05 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311616+0000", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118452 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311732+0000", 2024-09-13T14:33:09.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115845 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:178585 6.6 6.a2bd486 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.311894+0000", 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0060237040000000004, 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068079120000000002, 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.390 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 178585 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311894+0000", 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311894+0000", 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311895+0000", 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0100000000000004e-07 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311898+0000", 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7629999999999998e-06 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311908+0000", 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4520000000000003e-06 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.311915+0000", 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7700000000000004e-06 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.393 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178567 6.c e196/173)", 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315484+0000", 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0024336039999999998, 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032595440000000001, 2024-09-13T14:33:09.394 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315484+0000", 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315484+0000", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315485+0000", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.265e-06 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315490+0000", 2024-09-13T14:33:09.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9389999999999996e-06 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315507+0000", 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6730000000000001e-05 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315515+0000", 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0169999999999996e-06 2024-09-13T14:33:09.398 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178567 6.c e196/173)", 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315537+0000", 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0023802290000000002, 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032638020000000001, 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.399 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315537+0000", 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315537+0000", 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315538+0000", 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2899999999999999e-07 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315540+0000", 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5380000000000001e-06 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315547+0000", 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8469999999999998e-06 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315551+0000", 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3120000000000004e-06 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.402 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35349.0:178584 6.6 e196/173)", 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:09.315587+0000", 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0023308399999999998, 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00324961, 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315587+0000", 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315587+0000", 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315587+0000", 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9999999999999998e-07 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:33:09.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315589+0000", 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2089999999999999e-06 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315596+0000", 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9569999999999996e-06 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:09.315601+0000", 2024-09-13T14:33:09.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2300000000000002e-06 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 18 2024-09-13T14:33:09.406 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:33:09.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:09.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:09.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:09.661 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:09.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:09 smithi017 ceph-mon[103001]: pgmap v565: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:09 smithi017 ceph-mon[107556]: pgmap v565: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:09 smithi189 ceph-mon[90992]: pgmap v565: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:33:09.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:09.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:09.889 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:33:09.891 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:33:09.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:09.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:09.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:10.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:10.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:10.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:10.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:10.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:10.472 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:10.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:10.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:10.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:10.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:33:10.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:10.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:10 smithi189 ceph-mon[90992]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:10.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:10.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:10.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:10.917 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:10.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:10.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:10 smithi017 ceph-mon[103001]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:10 smithi017 ceph-mon[107556]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:11.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:33:11.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:11.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:11.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:11.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:11.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:11.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:11.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:11.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:11.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:11.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:11 smithi189 ceph-mon[90992]: pgmap v566: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:33:11.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:11.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:11.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:11 smithi017 ceph-mon[103001]: pgmap v566: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:33:11.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:11 smithi017 ceph-mon[107556]: pgmap v566: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:33:11.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:11.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:12.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:12.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:12.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:12.237 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":565,"stamp":"2024-09-13T14:33:07.884176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4896331749,"num_objects":74668,"num_object_clones":0,"num_object_copies":224004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74668,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":175487,"num_write_kb":7987600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133012,"ondisk_log_size":133012,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16862448,"kb_used_data":14422448,"kb_used_omap":35,"kb_used_meta":2439772,"kb_avail":733098768,"statfs":{"total":767960285184,"available":750693138432,"internally_reserved":0,"allocated":14768586752,"data_stored":14760469047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":2498327545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,4,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-539885568,"num_objects":-8238,"num_object_clones":0,"num_object_copies":-24714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8238,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002555"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":259,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.468087+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:32:58.468087+0000","last_peered":"2024-09-13T14:32:58.468087+0000","last_clean":"2024-09-13T14:32:58.468087+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:32:58.468087+0000","last_undegraded":"2024-09-13T14:32:58.468087+0000","last_fullsized":"2024-09-13T14:32:58.468087+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":162,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.536489+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:32:40.536489+0000","last_peered":"2024-09-13T14:32:40.536489+0000","last_clean":"2024-09-13T14:32:40.536489+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:32:40.536489+0000","last_undegraded":"2024-09-13T14:32:40.536489+0000","last_fullsized":"2024-09-13T14:32:40.536489+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":159,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539685+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:32:40.539685+0000","last_peered":"2024-09-13T14:32:40.539685+0000","last_clean":"2024-09-13T14:32:40.539685+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:32:40.539685+0000","last_undegraded":"2024-09-13T14:32:40.539685+0000","last_fullsized":"2024-09-13T14:32:40.539685+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":255,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.474375+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:32:58.474375+0000","last_peered":"2024-09-13T14:32:58.474375+0000","last_clean":"2024-09-13T14:32:58.474375+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:32:58.474375+0000","last_undegraded":"2024-09-13T14:32:58.474375+0000","last_fullsized":"2024-09-13T14:32:58.474375+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":256,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473674+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:32:58.473674+0000","last_peered":"2024-09-13T14:32:58.473674+0000","last_clean":"2024-09-13T14:32:58.473674+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:32:58.473674+0000","last_undegraded":"2024-09-13T14:32:58.473674+0000","last_fullsized":"2024-09-13T14:32:58.473674+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531525+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:32:40.531525+0000","last_peered":"2024-09-13T14:32:40.531525+0000","last_clean":"2024-09-13T14:32:40.531525+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:32:40.531525+0000","last_undegraded":"2024-09-13T14:32:40.531525+0000","last_fullsized":"2024-09-13T14:32:40.531525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":259,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.476539+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:32:58.476539+0000","last_peered":"2024-09-13T14:32:58.476539+0000","last_clean":"2024-09-13T14:32:58.476539+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:32:58.476539+0000","last_undegraded":"2024-09-13T14:32:58.476539+0000","last_fullsized":"2024-09-13T14:32:58.476539+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":168,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527314+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:32:40.527314+0000","last_peered":"2024-09-13T14:32:40.527314+0000","last_clean":"2024-09-13T14:32:40.527314+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:32:40.527314+0000","last_undegraded":"2024-09-13T14:32:40.527314+0000","last_fullsized":"2024-09-13T14:32:40.527314+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"196'10643","reported_seq":10674,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:02.989593+0000","last_change":"2024-09-13T14:30:27.500678+0000","last_active":"2024-09-13T14:33:02.989593+0000","last_peered":"2024-09-13T14:33:02.989593+0000","last_clean":"2024-09-13T14:33:02.989593+0000","last_became_active":"2024-09-13T14:30:27.500235+0000","last_became_peered":"2024-09-13T14:30:27.500235+0000","last_unstale":"2024-09-13T14:33:02.989593+0000","last_undegraded":"2024-09-13T14:33:02.989593+0000","last_fullsized":"2024-09-13T14:33:02.989593+0000","mapping_epoch":173,"log_start":"177'600","ondisk_log_start":"177'600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10043,"log_dups_size":0,"ondisk_log_size":10043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:11.965913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304807936,"num_objects":4651,"num_object_clones":0,"num_object_copies":13953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10643,"num_write_kb":489408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":161,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.532368+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:32:40.532368+0000","last_peered":"2024-09-13T14:32:40.532368+0000","last_clean":"2024-09-13T14:32:40.532368+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:32:40.532368+0000","last_undegraded":"2024-09-13T14:32:40.532368+0000","last_fullsized":"2024-09-13T14:32:40.532368+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":142,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539376+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:32:40.539376+0000","last_peered":"2024-09-13T14:32:40.539376+0000","last_clean":"2024-09-13T14:32:40.539376+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:32:40.539376+0000","last_undegraded":"2024-09-13T14:32:40.539376+0000","last_fullsized":"2024-09-13T14:32:40.539376+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"196'10769","reported_seq":10827,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:04.335395+0000","last_change":"2024-09-13T14:30:27.502589+0000","last_active":"2024-09-13T14:33:04.335395+0000","last_peered":"2024-09-13T14:33:04.335395+0000","last_clean":"2024-09-13T14:33:04.335395+0000","last_became_active":"2024-09-13T14:30:27.502327+0000","last_became_peered":"2024-09-13T14:30:27.502327+0000","last_unstale":"2024-09-13T14:33:04.335395+0000","last_undegraded":"2024-09-13T14:33:04.335395+0000","last_fullsized":"2024-09-13T14:33:04.335395+0000","mapping_epoch":173,"log_start":"182'4200","ondisk_log_start":"182'4200","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:12.990070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304807936,"num_objects":4651,"num_object_clones":0,"num_object_copies":13953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10769,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":162,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.532500+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:32:40.532500+0000","last_peered":"2024-09-13T14:32:40.532500+0000","last_clean":"2024-09-13T14:32:40.532500+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:32:40.532500+0000","last_undegraded":"2024-09-13T14:32:40.532500+0000","last_fullsized":"2024-09-13T14:32:40.532500+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":177,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538669+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:32:40.538669+0000","last_peered":"2024-09-13T14:32:40.538669+0000","last_clean":"2024-09-13T14:32:40.538669+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:32:40.538669+0000","last_undegraded":"2024-09-13T14:32:40.538669+0000","last_fullsized":"2024-09-13T14:32:40.538669+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"196'10855","reported_seq":10886,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:02.965552+0000","last_change":"2024-09-13T14:30:27.500921+0000","last_active":"2024-09-13T14:33:02.965552+0000","last_peered":"2024-09-13T14:33:02.965552+0000","last_clean":"2024-09-13T14:33:02.965552+0000","last_became_active":"2024-09-13T14:30:27.500457+0000","last_became_peered":"2024-09-13T14:30:27.500457+0000","last_unstale":"2024-09-13T14:33:02.965552+0000","last_undegraded":"2024-09-13T14:33:02.965552+0000","last_fullsized":"2024-09-13T14:33:02.965552+0000","mapping_epoch":173,"log_start":"180'3300","ondisk_log_start":"180'3300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7555,"log_dups_size":0,"ondisk_log_size":7555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:09.832130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310837248,"num_objects":4743,"num_object_clones":0,"num_object_copies":14229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10855,"num_write_kb":499136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":144,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.532224+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:32:40.532224+0000","last_peered":"2024-09-13T14:32:40.532224+0000","last_clean":"2024-09-13T14:32:40.532224+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:32:40.532224+0000","last_undegraded":"2024-09-13T14:32:40.532224+0000","last_fullsized":"2024-09-13T14:32:40.532224+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":159,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527981+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:32:40.527981+0000","last_peered":"2024-09-13T14:32:40.527981+0000","last_clean":"2024-09-13T14:32:40.527981+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:32:40.527981+0000","last_undegraded":"2024-09-13T14:32:40.527981+0000","last_fullsized":"2024-09-13T14:32:40.527981+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"196'11028","reported_seq":11086,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:07.554075+0000","last_change":"2024-09-13T14:30:27.501287+0000","last_active":"2024-09-13T14:33:07.554075+0000","last_peered":"2024-09-13T14:33:07.554075+0000","last_clean":"2024-09-13T14:33:07.554075+0000","last_became_active":"2024-09-13T14:30:27.500947+0000","last_became_peered":"2024-09-13T14:30:27.500947+0000","last_unstale":"2024-09-13T14:33:07.554075+0000","last_undegraded":"2024-09-13T14:33:07.554075+0000","last_fullsized":"2024-09-13T14:33:07.554075+0000","mapping_epoch":173,"log_start":"178'1900","ondisk_log_start":"178'1900","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:44:23.541886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293208064,"num_objects":4474,"num_object_clones":0,"num_object_copies":13422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11028,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539520+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:32:40.539520+0000","last_peered":"2024-09-13T14:32:40.539520+0000","last_clean":"2024-09-13T14:32:40.539520+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:32:40.539520+0000","last_undegraded":"2024-09-13T14:32:40.539520+0000","last_fullsized":"2024-09-13T14:32:40.539520+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":257,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.475385+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:32:58.475385+0000","last_peered":"2024-09-13T14:32:58.475385+0000","last_clean":"2024-09-13T14:32:58.475385+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:32:58.475385+0000","last_undegraded":"2024-09-13T14:32:58.475385+0000","last_fullsized":"2024-09-13T14:32:58.475385+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"196'10886","reported_seq":10917,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:03.029627+0000","last_change":"2024-09-13T14:30:27.501830+0000","last_active":"2024-09-13T14:33:03.029627+0000","last_peered":"2024-09-13T14:33:03.029627+0000","last_clean":"2024-09-13T14:33:03.029627+0000","last_became_active":"2024-09-13T14:30:27.501398+0000","last_became_peered":"2024-09-13T14:30:27.501398+0000","last_unstale":"2024-09-13T14:33:03.029627+0000","last_undegraded":"2024-09-13T14:33:03.029627+0000","last_fullsized":"2024-09-13T14:33:03.029627+0000","mapping_epoch":173,"log_start":"180'3500","ondisk_log_start":"180'3500","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:52.275184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316276736,"num_objects":4826,"num_object_clones":0,"num_object_copies":14478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10886,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":307,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531380+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:32:40.531380+0000","last_peered":"2024-09-13T14:32:40.531380+0000","last_clean":"2024-09-13T14:32:40.531380+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:32:40.531380+0000","last_undegraded":"2024-09-13T14:32:40.531380+0000","last_fullsized":"2024-09-13T14:32:40.531380+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":258,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.467739+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:32:58.467739+0000","last_peered":"2024-09-13T14:32:58.467739+0000","last_clean":"2024-09-13T14:32:58.467739+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:32:58.467739+0000","last_undegraded":"2024-09-13T14:32:58.467739+0000","last_fullsized":"2024-09-13T14:32:58.467739+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538960+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:32:40.538960+0000","last_peered":"2024-09-13T14:32:40.538960+0000","last_clean":"2024-09-13T14:32:40.538960+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:32:40.538960+0000","last_undegraded":"2024-09-13T14:32:40.538960+0000","last_fullsized":"2024-09-13T14:32:40.538960+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"196'11059","reported_seq":11117,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:06.291687+0000","last_change":"2024-09-13T14:30:27.505193+0000","last_active":"2024-09-13T14:33:06.291687+0000","last_peered":"2024-09-13T14:33:06.291687+0000","last_clean":"2024-09-13T14:33:06.291687+0000","last_became_active":"2024-09-13T14:30:27.504909+0000","last_became_peered":"2024-09-13T14:30:27.504909+0000","last_unstale":"2024-09-13T14:33:06.291687+0000","last_undegraded":"2024-09-13T14:33:06.291687+0000","last_fullsized":"2024-09-13T14:33:06.291687+0000","mapping_epoch":173,"log_start":"178'1600","ondisk_log_start":"178'1600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9459,"log_dups_size":0,"ondisk_log_size":9459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:40.244032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299171840,"num_objects":4565,"num_object_clones":0,"num_object_copies":13695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11059,"num_write_kb":499968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":54,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539300+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:32:40.539300+0000","last_peered":"2024-09-13T14:32:40.539300+0000","last_clean":"2024-09-13T14:32:40.539300+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:32:40.539300+0000","last_undegraded":"2024-09-13T14:32:40.539300+0000","last_fullsized":"2024-09-13T14:32:40.539300+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527865+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:32:40.527865+0000","last_peered":"2024-09-13T14:32:40.527865+0000","last_clean":"2024-09-13T14:32:40.527865+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:32:40.527865+0000","last_undegraded":"2024-09-13T14:32:40.527865+0000","last_fullsized":"2024-09-13T14:32:40.527865+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539135+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:32:40.539135+0000","last_peered":"2024-09-13T14:32:40.539135+0000","last_clean":"2024-09-13T14:32:40.539135+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:32:40.539135+0000","last_undegraded":"2024-09-13T14:32:40.539135+0000","last_fullsized":"2024-09-13T14:32:40.539135+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"196'10930","reported_seq":10988,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:04.370351+0000","last_change":"2024-09-13T14:30:27.503642+0000","last_active":"2024-09-13T14:33:04.370351+0000","last_peered":"2024-09-13T14:33:04.370351+0000","last_clean":"2024-09-13T14:33:04.370351+0000","last_became_active":"2024-09-13T14:30:27.503347+0000","last_became_peered":"2024-09-13T14:30:27.503347+0000","last_unstale":"2024-09-13T14:33:04.370351+0000","last_undegraded":"2024-09-13T14:33:04.370351+0000","last_fullsized":"2024-09-13T14:33:04.370351+0000","mapping_epoch":173,"log_start":"182'4400","ondisk_log_start":"182'4400","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:40.203221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10930,"num_write_kb":498368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":161,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538700+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:32:40.538700+0000","last_peered":"2024-09-13T14:32:40.538700+0000","last_clean":"2024-09-13T14:32:40.538700+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:32:40.538700+0000","last_undegraded":"2024-09-13T14:32:40.538700+0000","last_fullsized":"2024-09-13T14:32:40.538700+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539242+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:32:40.539242+0000","last_peered":"2024-09-13T14:32:40.539242+0000","last_clean":"2024-09-13T14:32:40.539242+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:32:40.539242+0000","last_undegraded":"2024-09-13T14:32:40.539242+0000","last_fullsized":"2024-09-13T14:32:40.539242+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"196'10822","reported_seq":10854,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:03.003972+0000","last_change":"2024-09-13T14:30:27.499796+0000","last_active":"2024-09-13T14:33:03.003972+0000","last_peered":"2024-09-13T14:33:03.003972+0000","last_clean":"2024-09-13T14:33:03.003972+0000","last_became_active":"2024-09-13T14:30:27.499475+0000","last_became_peered":"2024-09-13T14:30:27.499475+0000","last_unstale":"2024-09-13T14:33:03.003972+0000","last_undegraded":"2024-09-13T14:33:03.003972+0000","last_fullsized":"2024-09-13T14:33:03.003972+0000","mapping_epoch":173,"log_start":"177'800","ondisk_log_start":"177'800","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:07.838538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310968344,"num_objects":4746,"num_object_clones":0,"num_object_copies":14238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4746,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10822,"num_write_kb":498113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":47,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539544+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:32:40.539544+0000","last_peered":"2024-09-13T14:32:40.539544+0000","last_clean":"2024-09-13T14:32:40.539544+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:32:40.539544+0000","last_undegraded":"2024-09-13T14:32:40.539544+0000","last_fullsized":"2024-09-13T14:32:40.539544+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":261,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479951+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:32:58.479951+0000","last_peered":"2024-09-13T14:32:58.479951+0000","last_clean":"2024-09-13T14:32:58.479951+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:32:58.479951+0000","last_undegraded":"2024-09-13T14:32:58.479951+0000","last_fullsized":"2024-09-13T14:32:58.479951+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":246,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.480832+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:32:58.480832+0000","last_peered":"2024-09-13T14:32:58.480832+0000","last_clean":"2024-09-13T14:32:58.480832+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:32:58.480832+0000","last_undegraded":"2024-09-13T14:32:58.480832+0000","last_fullsized":"2024-09-13T14:32:58.480832+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"196'11073","reported_seq":11104,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:02.932292+0000","last_change":"2024-09-13T14:30:27.505472+0000","last_active":"2024-09-13T14:33:02.932292+0000","last_peered":"2024-09-13T14:33:02.932292+0000","last_clean":"2024-09-13T14:33:02.932292+0000","last_became_active":"2024-09-13T14:30:27.505098+0000","last_became_peered":"2024-09-13T14:30:27.505098+0000","last_unstale":"2024-09-13T14:33:02.932292+0000","last_undegraded":"2024-09-13T14:33:02.932292+0000","last_fullsized":"2024-09-13T14:33:02.932292+0000","mapping_epoch":173,"log_start":"177'1000","ondisk_log_start":"177'1000","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10073,"log_dups_size":0,"ondisk_log_size":10073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:45.218708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310706176,"num_objects":4741,"num_object_clones":0,"num_object_copies":14223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11073,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":298,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531408+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:32:40.531408+0000","last_peered":"2024-09-13T14:32:40.531408+0000","last_clean":"2024-09-13T14:32:40.531408+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:32:40.531408+0000","last_undegraded":"2024-09-13T14:32:40.531408+0000","last_fullsized":"2024-09-13T14:32:40.531408+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":761,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539307+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:32:40.539307+0000","last_peered":"2024-09-13T14:32:40.539307+0000","last_clean":"2024-09-13T14:32:40.539307+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:32:40.539307+0000","last_undegraded":"2024-09-13T14:32:40.539307+0000","last_fullsized":"2024-09-13T14:32:40.539307+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":214,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.467931+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:32:58.467931+0000","last_peered":"2024-09-13T14:32:58.467931+0000","last_clean":"2024-09-13T14:32:58.467931+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:32:58.467931+0000","last_undegraded":"2024-09-13T14:32:58.467931+0000","last_fullsized":"2024-09-13T14:32:58.467931+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":142,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527759+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:32:40.527759+0000","last_peered":"2024-09-13T14:32:40.527759+0000","last_clean":"2024-09-13T14:32:40.527759+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:32:40.527759+0000","last_undegraded":"2024-09-13T14:32:40.527759+0000","last_fullsized":"2024-09-13T14:32:40.527759+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"196'11259","reported_seq":11317,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:07.373221+0000","last_change":"2024-09-13T14:30:27.505328+0000","last_active":"2024-09-13T14:33:07.373221+0000","last_peered":"2024-09-13T14:33:07.373221+0000","last_clean":"2024-09-13T14:33:07.373221+0000","last_became_active":"2024-09-13T14:30:27.505015+0000","last_became_peered":"2024-09-13T14:30:27.505015+0000","last_unstale":"2024-09-13T14:33:07.373221+0000","last_undegraded":"2024-09-13T14:33:07.373221+0000","last_fullsized":"2024-09-13T14:33:07.373221+0000","mapping_epoch":173,"log_start":"179'3300","ondisk_log_start":"179'3300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:42.013686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292880384,"num_objects":4469,"num_object_clones":0,"num_object_copies":13407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11259,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":91,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.482568+0000","last_change":"2024-09-13T14:30:39.822827+0000","last_active":"2024-09-13T14:32:58.482568+0000","last_peered":"2024-09-13T14:32:58.482568+0000","last_clean":"2024-09-13T14:32:58.482568+0000","last_became_active":"2024-09-13T14:30:11.437578+0000","last_became_peered":"2024-09-13T14:30:11.437578+0000","last_unstale":"2024-09-13T14:32:58.482568+0000","last_undegraded":"2024-09-13T14:32:58.482568+0000","last_fullsized":"2024-09-13T14:32:58.482568+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.822747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.822747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:44.769254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":569,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531325+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:32:40.531325+0000","last_peered":"2024-09-13T14:32:40.531325+0000","last_clean":"2024-09-13T14:32:40.531325+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:32:40.531325+0000","last_undegraded":"2024-09-13T14:32:40.531325+0000","last_fullsized":"2024-09-13T14:32:40.531325+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":256,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.474028+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:32:58.474028+0000","last_peered":"2024-09-13T14:32:58.474028+0000","last_clean":"2024-09-13T14:32:58.474028+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:32:58.474028+0000","last_undegraded":"2024-09-13T14:32:58.474028+0000","last_fullsized":"2024-09-13T14:32:58.474028+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473835+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:32:58.473835+0000","last_peered":"2024-09-13T14:32:58.473835+0000","last_clean":"2024-09-13T14:32:58.473835+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:32:58.473835+0000","last_undegraded":"2024-09-13T14:32:58.473835+0000","last_fullsized":"2024-09-13T14:32:58.473835+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"196'10690","reported_seq":10721,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:02.981576+0000","last_change":"2024-09-13T14:30:27.500479+0000","last_active":"2024-09-13T14:33:02.981576+0000","last_peered":"2024-09-13T14:33:02.981576+0000","last_clean":"2024-09-13T14:33:02.981576+0000","last_became_active":"2024-09-13T14:30:27.500110+0000","last_became_peered":"2024-09-13T14:30:27.500110+0000","last_unstale":"2024-09-13T14:33:02.981576+0000","last_undegraded":"2024-09-13T14:33:02.981576+0000","last_fullsized":"2024-09-13T14:33:02.981576+0000","mapping_epoch":173,"log_start":"179'3100","ondisk_log_start":"179'3100","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7590,"log_dups_size":0,"ondisk_log_size":7590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:10.174744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10690,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":307,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531498+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:32:40.531498+0000","last_peered":"2024-09-13T14:32:40.531498+0000","last_clean":"2024-09-13T14:32:40.531498+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:32:40.531498+0000","last_undegraded":"2024-09-13T14:32:40.531498+0000","last_fullsized":"2024-09-13T14:32:40.531498+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":685,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.478381+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:32:58.478381+0000","last_peered":"2024-09-13T14:32:58.478381+0000","last_clean":"2024-09-13T14:32:58.478381+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:32:58.478381+0000","last_undegraded":"2024-09-13T14:32:58.478381+0000","last_fullsized":"2024-09-13T14:32:58.478381+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":247,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479152+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:32:58.479152+0000","last_peered":"2024-09-13T14:32:58.479152+0000","last_clean":"2024-09-13T14:32:58.479152+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:32:58.479152+0000","last_undegraded":"2024-09-13T14:32:58.479152+0000","last_fullsized":"2024-09-13T14:32:58.479152+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":262,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.481394+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:32:58.481394+0000","last_peered":"2024-09-13T14:32:58.481394+0000","last_clean":"2024-09-13T14:32:58.481394+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:32:58.481394+0000","last_undegraded":"2024-09-13T14:32:58.481394+0000","last_fullsized":"2024-09-13T14:32:58.481394+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"196'10860","reported_seq":10918,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:04.282012+0000","last_change":"2024-09-13T14:30:27.501260+0000","last_active":"2024-09-13T14:33:04.282012+0000","last_peered":"2024-09-13T14:33:04.282012+0000","last_clean":"2024-09-13T14:33:04.282012+0000","last_became_active":"2024-09-13T14:30:27.500950+0000","last_became_peered":"2024-09-13T14:30:27.500950+0000","last_unstale":"2024-09-13T14:33:04.282012+0000","last_undegraded":"2024-09-13T14:33:04.282012+0000","last_fullsized":"2024-09-13T14:33:04.282012+0000","mapping_epoch":173,"log_start":"182'4300","ondisk_log_start":"182'4300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:54.878021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10860,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":47,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539459+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:32:40.539459+0000","last_peered":"2024-09-13T14:32:40.539459+0000","last_clean":"2024-09-13T14:32:40.539459+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:32:40.539459+0000","last_undegraded":"2024-09-13T14:32:40.539459+0000","last_fullsized":"2024-09-13T14:32:40.539459+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":810,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539624+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:32:40.539624+0000","last_peered":"2024-09-13T14:32:40.539624+0000","last_clean":"2024-09-13T14:32:40.539624+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:32:40.539624+0000","last_undegraded":"2024-09-13T14:32:40.539624+0000","last_fullsized":"2024-09-13T14:32:40.539624+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539029+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:32:40.539029+0000","last_peered":"2024-09-13T14:32:40.539029+0000","last_clean":"2024-09-13T14:32:40.539029+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:32:40.539029+0000","last_undegraded":"2024-09-13T14:32:40.539029+0000","last_fullsized":"2024-09-13T14:32:40.539029+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":257,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479619+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:32:58.479619+0000","last_peered":"2024-09-13T14:32:58.479619+0000","last_clean":"2024-09-13T14:32:58.479619+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:32:58.479619+0000","last_undegraded":"2024-09-13T14:32:58.479619+0000","last_fullsized":"2024-09-13T14:32:58.479619+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"196'10730","reported_seq":10788,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:07.567371+0000","last_change":"2024-09-13T14:30:27.502045+0000","last_active":"2024-09-13T14:33:07.567371+0000","last_peered":"2024-09-13T14:33:07.567371+0000","last_clean":"2024-09-13T14:33:07.567371+0000","last_became_active":"2024-09-13T14:30:27.501711+0000","last_became_peered":"2024-09-13T14:30:27.501711+0000","last_unstale":"2024-09-13T14:33:07.567371+0000","last_undegraded":"2024-09-13T14:33:07.567371+0000","last_fullsized":"2024-09-13T14:33:07.567371+0000","mapping_epoch":173,"log_start":"178'1600","ondisk_log_start":"178'1600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9130,"log_dups_size":0,"ondisk_log_size":9130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:21.043630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291241984,"num_objects":4444,"num_object_clones":0,"num_object_copies":13332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10730,"num_write_kb":485568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":195,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527398+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:32:40.527398+0000","last_peered":"2024-09-13T14:32:40.527398+0000","last_clean":"2024-09-13T14:32:40.527398+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:32:40.527398+0000","last_undegraded":"2024-09-13T14:32:40.527398+0000","last_fullsized":"2024-09-13T14:32:40.527398+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539760+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:32:40.539760+0000","last_peered":"2024-09-13T14:32:40.539760+0000","last_clean":"2024-09-13T14:32:40.539760+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:32:40.539760+0000","last_undegraded":"2024-09-13T14:32:40.539760+0000","last_fullsized":"2024-09-13T14:32:40.539760+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":856,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479798+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:32:58.479798+0000","last_peered":"2024-09-13T14:32:58.479798+0000","last_clean":"2024-09-13T14:32:58.479798+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:32:58.479798+0000","last_undegraded":"2024-09-13T14:32:58.479798+0000","last_fullsized":"2024-09-13T14:32:58.479798+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":235,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.478656+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:32:58.478656+0000","last_peered":"2024-09-13T14:32:58.478656+0000","last_clean":"2024-09-13T14:32:58.478656+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:32:58.478656+0000","last_undegraded":"2024-09-13T14:32:58.478656+0000","last_fullsized":"2024-09-13T14:32:58.478656+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"196'11069","reported_seq":11127,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:07.412227+0000","last_change":"2024-09-13T14:30:27.500989+0000","last_active":"2024-09-13T14:33:07.412227+0000","last_peered":"2024-09-13T14:33:07.412227+0000","last_clean":"2024-09-13T14:33:07.412227+0000","last_became_active":"2024-09-13T14:30:27.500671+0000","last_became_peered":"2024-09-13T14:30:27.500671+0000","last_unstale":"2024-09-13T14:33:07.412227+0000","last_undegraded":"2024-09-13T14:33:07.412227+0000","last_fullsized":"2024-09-13T14:33:07.412227+0000","mapping_epoch":173,"log_start":"179'3100","ondisk_log_start":"179'3100","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7969,"log_dups_size":0,"ondisk_log_size":7969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:59.179207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11069,"num_write_kb":503808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":195,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527184+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:32:40.527184+0000","last_peered":"2024-09-13T14:32:40.527184+0000","last_clean":"2024-09-13T14:32:40.527184+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:32:40.527184+0000","last_undegraded":"2024-09-13T14:32:40.527184+0000","last_fullsized":"2024-09-13T14:32:40.527184+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479708+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:32:58.479708+0000","last_peered":"2024-09-13T14:32:58.479708+0000","last_clean":"2024-09-13T14:32:58.479708+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:32:58.479708+0000","last_undegraded":"2024-09-13T14:32:58.479708+0000","last_fullsized":"2024-09-13T14:32:58.479708+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":506,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.532243+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:32:40.532243+0000","last_peered":"2024-09-13T14:32:40.532243+0000","last_clean":"2024-09-13T14:32:40.532243+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:32:40.532243+0000","last_undegraded":"2024-09-13T14:32:40.532243+0000","last_fullsized":"2024-09-13T14:32:40.532243+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":246,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.475313+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:32:58.475313+0000","last_peered":"2024-09-13T14:32:58.475313+0000","last_clean":"2024-09-13T14:32:58.475313+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:32:58.475313+0000","last_undegraded":"2024-09-13T14:32:58.475313+0000","last_fullsized":"2024-09-13T14:32:58.475313+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"196'10958","reported_seq":10989,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:02.965135+0000","last_change":"2024-09-13T14:30:27.502612+0000","last_active":"2024-09-13T14:33:02.965135+0000","last_peered":"2024-09-13T14:33:02.965135+0000","last_clean":"2024-09-13T14:33:02.965135+0000","last_became_active":"2024-09-13T14:30:27.501659+0000","last_became_peered":"2024-09-13T14:30:27.501659+0000","last_unstale":"2024-09-13T14:33:02.965135+0000","last_undegraded":"2024-09-13T14:33:02.965135+0000","last_fullsized":"2024-09-13T14:33:02.965135+0000","mapping_epoch":173,"log_start":"180'3600","ondisk_log_start":"180'3600","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:35:47.542650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311689216,"num_objects":4756,"num_object_clones":0,"num_object_copies":14268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10958,"num_write_kb":502848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":47,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538966+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:32:40.538966+0000","last_peered":"2024-09-13T14:32:40.538966+0000","last_clean":"2024-09-13T14:32:40.538966+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:32:40.538966+0000","last_undegraded":"2024-09-13T14:32:40.538966+0000","last_fullsized":"2024-09-13T14:32:40.538966+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":258,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.476724+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:32:58.476724+0000","last_peered":"2024-09-13T14:32:58.476724+0000","last_clean":"2024-09-13T14:32:58.476724+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:32:58.476724+0000","last_undegraded":"2024-09-13T14:32:58.476724+0000","last_fullsized":"2024-09-13T14:32:58.476724+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":590,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479859+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:32:58.479859+0000","last_peered":"2024-09-13T14:32:58.479859+0000","last_clean":"2024-09-13T14:32:58.479859+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:32:58.479859+0000","last_undegraded":"2024-09-13T14:32:58.479859+0000","last_fullsized":"2024-09-13T14:32:58.479859+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":255,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479713+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:32:58.479713+0000","last_peered":"2024-09-13T14:32:58.479713+0000","last_clean":"2024-09-13T14:32:58.479713+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:32:58.479713+0000","last_undegraded":"2024-09-13T14:32:58.479713+0000","last_fullsized":"2024-09-13T14:32:58.479713+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"196'10949","reported_seq":10980,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:33:03.007237+0000","last_change":"2024-09-13T14:30:27.501009+0000","last_active":"2024-09-13T14:33:03.007237+0000","last_peered":"2024-09-13T14:33:03.007237+0000","last_clean":"2024-09-13T14:33:03.007237+0000","last_became_active":"2024-09-13T14:30:27.500481+0000","last_became_peered":"2024-09-13T14:30:27.500481+0000","last_unstale":"2024-09-13T14:33:03.007237+0000","last_undegraded":"2024-09-13T14:33:03.007237+0000","last_fullsized":"2024-09-13T14:33:03.007237+0000","mapping_epoch":173,"log_start":"180'3400","ondisk_log_start":"180'3400","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:30.536488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046208,"num_objects":4853,"num_object_clones":0,"num_object_copies":14559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10949,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":96,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.467929+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:32:58.467929+0000","last_peered":"2024-09-13T14:32:58.467929+0000","last_clean":"2024-09-13T14:32:58.467929+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:32:58.467929+0000","last_undegraded":"2024-09-13T14:32:58.467929+0000","last_fullsized":"2024-09-13T14:32:58.467929+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":260,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473103+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:32:58.473103+0000","last_peered":"2024-09-13T14:32:58.473103+0000","last_clean":"2024-09-13T14:32:58.473103+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:32:58.473103+0000","last_undegraded":"2024-09-13T14:32:58.473103+0000","last_fullsized":"2024-09-13T14:32:58.473103+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":614,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527477+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:32:40.527477+0000","last_peered":"2024-09-13T14:32:40.527477+0000","last_clean":"2024-09-13T14:32:40.527477+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:32:40.527477+0000","last_undegraded":"2024-09-13T14:32:40.527477+0000","last_fullsized":"2024-09-13T14:32:40.527477+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538599+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:32:40.538599+0000","last_peered":"2024-09-13T14:32:40.538599+0000","last_clean":"2024-09-13T14:32:40.538599+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:32:40.538599+0000","last_undegraded":"2024-09-13T14:32:40.538599+0000","last_fullsized":"2024-09-13T14:32:40.538599+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538891+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:32:40.538891+0000","last_peered":"2024-09-13T14:32:40.538891+0000","last_clean":"2024-09-13T14:32:40.538891+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:32:40.538891+0000","last_undegraded":"2024-09-13T14:32:40.538891+0000","last_fullsized":"2024-09-13T14:32:40.538891+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":166,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539047+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:32:40.539047+0000","last_peered":"2024-09-13T14:32:40.539047+0000","last_clean":"2024-09-13T14:32:40.539047+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:32:40.539047+0000","last_undegraded":"2024-09-13T14:32:40.539047+0000","last_fullsized":"2024-09-13T14:32:40.539047+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":246,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.478521+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:32:58.478521+0000","last_peered":"2024-09-13T14:32:58.478521+0000","last_clean":"2024-09-13T14:32:58.478521+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:32:58.478521+0000","last_undegraded":"2024-09-13T14:32:58.478521+0000","last_fullsized":"2024-09-13T14:32:58.478521+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":163,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.531808+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:32:40.531808+0000","last_peered":"2024-09-13T14:32:40.531808+0000","last_clean":"2024-09-13T14:32:40.531808+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:32:40.531808+0000","last_undegraded":"2024-09-13T14:32:40.531808+0000","last_fullsized":"2024-09-13T14:32:40.531808+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":160,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539597+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:32:40.539597+0000","last_peered":"2024-09-13T14:32:40.539597+0000","last_clean":"2024-09-13T14:32:40.539597+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:32:40.539597+0000","last_undegraded":"2024-09-13T14:32:40.539597+0000","last_fullsized":"2024-09-13T14:32:40.539597+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.467899+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:32:58.467899+0000","last_peered":"2024-09-13T14:32:58.467899+0000","last_clean":"2024-09-13T14:32:58.467899+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:32:58.467899+0000","last_undegraded":"2024-09-13T14:32:58.467899+0000","last_fullsized":"2024-09-13T14:32:58.467899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":255,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473398+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:32:58.473398+0000","last_peered":"2024-09-13T14:32:58.473398+0000","last_clean":"2024-09-13T14:32:58.473398+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:32:58.473398+0000","last_undegraded":"2024-09-13T14:32:58.473398+0000","last_fullsized":"2024-09-13T14:32:58.473398+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479666+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:32:58.479666+0000","last_peered":"2024-09-13T14:32:58.479666+0000","last_clean":"2024-09-13T14:32:58.479666+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:32:58.479666+0000","last_undegraded":"2024-09-13T14:32:58.479666+0000","last_fullsized":"2024-09-13T14:32:58.479666+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.527438+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:32:40.527438+0000","last_peered":"2024-09-13T14:32:40.527438+0000","last_clean":"2024-09-13T14:32:40.527438+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:32:40.527438+0000","last_undegraded":"2024-09-13T14:32:40.527438+0000","last_fullsized":"2024-09-13T14:32:40.527438+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.479841+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:32:58.479841+0000","last_peered":"2024-09-13T14:32:58.479841+0000","last_clean":"2024-09-13T14:32:58.479841+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:32:58.479841+0000","last_undegraded":"2024-09-13T14:32:58.479841+0000","last_fullsized":"2024-09-13T14:32:58.479841+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473756+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:32:58.473756+0000","last_peered":"2024-09-13T14:32:58.473756+0000","last_clean":"2024-09-13T14:32:58.473756+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:32:58.473756+0000","last_undegraded":"2024-09-13T14:32:58.473756+0000","last_fullsized":"2024-09-13T14:32:58.473756+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":247,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.480925+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:32:58.480925+0000","last_peered":"2024-09-13T14:32:58.480925+0000","last_clean":"2024-09-13T14:32:58.480925+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:32:58.480925+0000","last_undegraded":"2024-09-13T14:32:58.480925+0000","last_fullsized":"2024-09-13T14:32:58.480925+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":259,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.468807+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:32:58.468807+0000","last_peered":"2024-09-13T14:32:58.468807+0000","last_clean":"2024-09-13T14:32:58.468807+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:32:58.468807+0000","last_undegraded":"2024-09-13T14:32:58.468807+0000","last_fullsized":"2024-09-13T14:32:58.468807+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":179,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538845+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:32:40.538845+0000","last_peered":"2024-09-13T14:32:40.538845+0000","last_clean":"2024-09-13T14:32:40.538845+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:32:40.538845+0000","last_undegraded":"2024-09-13T14:32:40.538845+0000","last_fullsized":"2024-09-13T14:32:40.538845+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":279,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.468920+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:32:58.468920+0000","last_peered":"2024-09-13T14:32:58.468920+0000","last_clean":"2024-09-13T14:32:58.468920+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:32:58.468920+0000","last_undegraded":"2024-09-13T14:32:58.468920+0000","last_fullsized":"2024-09-13T14:32:58.468920+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":248,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.475224+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:32:58.475224+0000","last_peered":"2024-09-13T14:32:58.475224+0000","last_clean":"2024-09-13T14:32:58.475224+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:32:58.475224+0000","last_undegraded":"2024-09-13T14:32:58.475224+0000","last_fullsized":"2024-09-13T14:32:58.475224+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538865+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:32:40.538865+0000","last_peered":"2024-09-13T14:32:40.538865+0000","last_clean":"2024-09-13T14:32:40.538865+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:32:40.538865+0000","last_undegraded":"2024-09-13T14:32:40.538865+0000","last_fullsized":"2024-09-13T14:32:40.538865+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":162,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539435+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:32:40.539435+0000","last_peered":"2024-09-13T14:32:40.539435+0000","last_clean":"2024-09-13T14:32:40.539435+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:32:40.539435+0000","last_undegraded":"2024-09-13T14:32:40.539435+0000","last_fullsized":"2024-09-13T14:32:40.539435+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538768+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:32:40.538768+0000","last_peered":"2024-09-13T14:32:40.538768+0000","last_clean":"2024-09-13T14:32:40.538768+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:32:40.538768+0000","last_undegraded":"2024-09-13T14:32:40.538768+0000","last_fullsized":"2024-09-13T14:32:40.538768+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":145,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.539761+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:32:40.539761+0000","last_peered":"2024-09-13T14:32:40.539761+0000","last_clean":"2024-09-13T14:32:40.539761+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:32:40.539761+0000","last_undegraded":"2024-09-13T14:32:40.539761+0000","last_fullsized":"2024-09-13T14:32:40.539761+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":280,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.467856+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:32:58.467856+0000","last_peered":"2024-09-13T14:32:58.467856+0000","last_clean":"2024-09-13T14:32:58.467856+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:32:58.467856+0000","last_undegraded":"2024-09-13T14:32:58.467856+0000","last_fullsized":"2024-09-13T14:32:58.467856+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":266,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.468305+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:32:58.468305+0000","last_peered":"2024-09-13T14:32:58.468305+0000","last_clean":"2024-09-13T14:32:58.468305+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:32:58.468305+0000","last_undegraded":"2024-09-13T14:32:58.468305+0000","last_fullsized":"2024-09-13T14:32:58.468305+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":195,"state":"active+clean","last_fresh":"2024-09-13T14:32:40.538541+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:32:40.538541+0000","last_peered":"2024-09-13T14:32:40.538541+0000","last_clean":"2024-09-13T14:32:40.538541+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:32:40.538541+0000","last_undegraded":"2024-09-13T14:32:40.538541+0000","last_fullsized":"2024-09-13T14:32:40.538541+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":260,"reported_epoch":196,"state":"active+clean","last_fresh":"2024-09-13T14:32:58.473305+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:32:58.473305+0000","last_peered":"2024-09-13T14:32:58.473305+0000","last_clean":"2024-09-13T14:32:58.473305+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:32:58.473305+0000","last_undegraded":"2024-09-13T14:32:58.473305+0000","last_fullsized":"2024-09-13T14:32:58.473305+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4891541528,"num_objects":74640,"num_object_clones":0,"num_object_copies":223920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74640,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174580,"num_write_kb":7974977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14644817920,"data_stored":14644805704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130880,"ondisk_log_size":130880,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397763,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2130492,"kb_used_data":1828980,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91614660,"statfs":{"total":95995035648,"available":93813411840,"internally_reserved":0,"allocated":1872875520,"data_stored":1871849319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593991,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1844780,"kb_used_data":1534948,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91900372,"statfs":{"total":95995035648,"available":94105980928,"internally_reserved":0,"allocated":1571786752,"data_stored":1570776241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790221,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821484,"kb_used_data":1515812,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91923668,"statfs":{"total":95995035648,"available":94129836032,"internally_reserved":0,"allocated":1552191488,"data_stored":1551177308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986449,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827640,"kb_used_data":1530288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91917512,"statfs":{"total":95995035648,"available":94123532288,"internally_reserved":0,"allocated":1567014912,"data_stored":1565992737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697007,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3347432,"kb_used_data":3011296,"kb_used_omap":4,"kb_used_meta":336059,"kb_avail":90397720,"statfs":{"total":95995035648,"available":92567265280,"internally_reserved":0,"allocated":3083567104,"data_stored":3082557069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":344124491},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860534,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2691948,"kb_used_data":2376548,"kb_used_omap":8,"kb_used_meta":315383,"kb_avail":91053204,"statfs":{"total":95995035648,"available":93238480896,"internally_reserved":0,"allocated":2433585152,"data_stored":2432571543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":322953036},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024061,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1763944,"kb_used_data":1472032,"kb_used_omap":7,"kb_used_meta":291832,"kb_avail":91981208,"statfs":{"total":95995035648,"available":94188756992,"internally_reserved":0,"allocated":1507360768,"data_stored":1506350134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":298836407},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187596,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1434728,"kb_used_data":1152544,"kb_used_omap":7,"kb_used_meta":282168,"kb_avail":92310424,"statfs":{"total":95995035648,"available":94525874176,"internally_reserved":0,"allocated":1180205056,"data_stored":1179194696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":288940363},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1166086144,"data_stored":1166082072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1484525568,"data_stored":1484521496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2410348544,"data_stored":2410348544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3050438656,"data_stored":3050438656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1557528576,"data_stored":1557528576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1544622080,"data_stored":1544618008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1565786112,"data_stored":1565786112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1865482240,"data_stored":1865482240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:33:12.241 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:33:12.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_1 pg_num 26 2024-09-13T14:33:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:12.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:12.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:12.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:12.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:12.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:12.508 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:12.508 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:12.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:12.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:12.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:12.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:12.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:12.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:12.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:12.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:13.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:13.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:13.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:13.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:13.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:13.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[103001]: pgmap v567: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:33:13.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:13.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:13.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[107556]: pgmap v567: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:33:13.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:13.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:13.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:13.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:13.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:13 smithi189 ceph-mon[90992]: pgmap v567: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:33:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:13.544 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:13.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:13.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:13.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:13.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:13.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:13.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:13.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:14.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:14.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:14.129 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:33:14.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:14.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:14.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:14.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1007984040' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:14 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1007984040' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:14 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:14.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1007984040' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:14 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:14.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:14.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:14.718 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:14.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:14.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:14.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:14.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:14.982 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:15.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:15.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:15.143 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:15.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:15.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:15.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:15.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[103001]: pgmap v568: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:33:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[103001]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:33:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[107556]: pgmap v568: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:33:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:15 smithi017 ceph-mon[107556]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:33:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:15 smithi189 ceph-mon[90992]: pgmap v568: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T14:33:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:15 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:15 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:15.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:15 smithi189 ceph-mon[90992]: osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:33:15.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:15.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:15.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:15.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:15.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:15.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:15.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:15.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:15.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:15.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:16.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:16.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:16.205 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:16.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:16.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:16.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:16.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:16.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:16.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:16.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:16.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:16.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:16.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:16.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:16.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:16.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:17.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:17.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:17.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:17.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:17.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:17 smithi017 ceph-mon[103001]: pgmap v570: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-13T14:33:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:17] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-13T14:33:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:17 smithi017 ceph-mon[107556]: pgmap v570: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-13T14:33:17.466 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:17 smithi189 ceph-mon[90992]: pgmap v570: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-09-13T14:33:17.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.596 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:17.697 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:17.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:17.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:17.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:17.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:17.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:17.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:17.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:18.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:18.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:18.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:18.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:18.380 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.406 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:18.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:18.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:18.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:18.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:18.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:18.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:18.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:18.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:33:18.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:18.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:18.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:19.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:19.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:19.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:19.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:19.333 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:19 smithi017 ceph-mon[103001]: pgmap v571: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:33:19.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:19 smithi017 ceph-mon[107556]: pgmap v571: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:33:19.392 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:33:19.393 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:33:19.393 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:33:19.393 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:33:19.393 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:33:19.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:19.441 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:19 smithi189 ceph-mon[90992]: pgmap v571: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-13T14:33:19.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:19.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:19.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:19.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:19.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:19.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:19.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:19.963 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:19.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:20.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:20.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:20.155 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 6 pg_num to 26 2024-09-13T14:33:20.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:20.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:20.305 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:33:20.305 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:33:20.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:20.355 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:33:20.355 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:33:20.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:20.393 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:33:20.393 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:33:20.393 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:33:20.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:33:20.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:33:20.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:33:20.471 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:33:20.472 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:33:20.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:33:20.508 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:33:20.508 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:33:20.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:20.544 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:33:20.544 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:33:20.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:33:20.579 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:33:20.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:20.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:20.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:33:20.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:20.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:20.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:20.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:20.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:21.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:21.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:21.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:21.173 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:21.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:21.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:21.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:21.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[103001]: pgmap v572: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[103001]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3797019438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[107556]: pgmap v572: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-09-13T14:33:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:33:21.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[107556]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:33:21.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3797019438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:21.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:21.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:21 smithi189 ceph-mon[90992]: pgmap v572: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-09-13T14:33:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/449852348' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:33:21.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:21 smithi189 ceph-mon[90992]: osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:33:21.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3797019438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:21.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:21.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:21.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:21.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:21.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:21.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:21.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:21.982 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:22.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:22.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:22.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:22.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:22.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:22.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:33:22.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:33:22.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:33:22.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:22.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:22.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:22.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:22.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:22.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:22.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:22.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:23.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:23.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:23.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:23.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:23.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:23.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[103001]: pgmap v574: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:33:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:33:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[103001]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:33:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[107556]: pgmap v574: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:33:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:33:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:23 smithi017 ceph-mon[107556]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:33:23.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.484 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:23.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:23 smithi189 ceph-mon[90992]: pgmap v574: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:33:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:33:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:23 smithi189 ceph-mon[90992]: osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:33:23.554 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:23.565 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:23.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:23.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:23.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:23.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:23.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:23.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:23.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:23.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:23.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:24.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:24.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:24.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:24.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:24.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:24.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[103001]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:33:24.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[103001]: 6.6 scrub starts 2024-09-13T14:33:24.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[103001]: 6.0 scrub starts 2024-09-13T14:33:24.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[107556]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:33:24.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[107556]: 6.6 scrub starts 2024-09-13T14:33:24.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:24 smithi017 ceph-mon[107556]: 6.0 scrub starts 2024-09-13T14:33:24.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:24.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:24.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:33:24.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:24 smithi189 ceph-mon[90992]: osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:33:24.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:24 smithi189 ceph-mon[90992]: 6.6 scrub starts 2024-09-13T14:33:24.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:24 smithi189 ceph-mon[90992]: 6.0 scrub starts 2024-09-13T14:33:24.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:24.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:24.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:24.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:24.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:24.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:25.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:25.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:25.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:25.224 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:25.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:25.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[103001]: 6.14 scrub starts 2024-09-13T14:33:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[103001]: pgmap v577: 109 pgs: 9 peering, 10 unknown, 90 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[103001]: 6.16 scrub starts 2024-09-13T14:33:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[103001]: 6.9 scrub starts 2024-09-13T14:33:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[107556]: 6.14 scrub starts 2024-09-13T14:33:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[107556]: pgmap v577: 109 pgs: 9 peering, 10 unknown, 90 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[107556]: 6.16 scrub starts 2024-09-13T14:33:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:25 smithi017 ceph-mon[107556]: 6.9 scrub starts 2024-09-13T14:33:25.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:25.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:25 smithi189 ceph-mon[90992]: 6.14 scrub starts 2024-09-13T14:33:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:25 smithi189 ceph-mon[90992]: pgmap v577: 109 pgs: 9 peering, 10 unknown, 90 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:25 smithi189 ceph-mon[90992]: 6.16 scrub starts 2024-09-13T14:33:25.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:25 smithi189 ceph-mon[90992]: 6.9 scrub starts 2024-09-13T14:33:25.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:25.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:25.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:25.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:25.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:25.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:25.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:26.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:26.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:26.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:26.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:26.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:26.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:26.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:26.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:26.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:26.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:26 smithi017 ceph-mon[103001]: 6.4 scrub starts 2024-09-13T14:33:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:26 smithi017 ceph-mon[103001]: 6.10 scrub starts 2024-09-13T14:33:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:26 smithi017 ceph-mon[107556]: 6.4 scrub starts 2024-09-13T14:33:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:26 smithi017 ceph-mon[107556]: 6.10 scrub starts 2024-09-13T14:33:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:26 smithi189 ceph-mon[90992]: 6.4 scrub starts 2024-09-13T14:33:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:26 smithi189 ceph-mon[90992]: 6.10 scrub starts 2024-09-13T14:33:26.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:26.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:26.647 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:26.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:26.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:26.809 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:26.842 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:26.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:26.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:26.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:27.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:27.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:27.249 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:27.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:27.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:27.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:27.430 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:27] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.51.0" 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[103001]: 6.11 scrub starts 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[103001]: pgmap v578: 109 pgs: 9 peering, 9 unknown, 91 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[103001]: 6.19 scrub starts 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[107556]: 6.11 scrub starts 2024-09-13T14:33:27.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[107556]: pgmap v578: 109 pgs: 9 peering, 9 unknown, 91 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:27.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[107556]: 6.19 scrub starts 2024-09-13T14:33:27.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:27 smithi189 ceph-mon[90992]: 6.11 scrub starts 2024-09-13T14:33:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:27 smithi189 ceph-mon[90992]: pgmap v578: 109 pgs: 9 peering, 9 unknown, 91 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:33:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:27 smithi189 ceph-mon[90992]: 6.19 scrub starts 2024-09-13T14:33:27.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:27.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:27.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:27.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:27.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:27.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:27.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:27.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:27.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:27.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:28.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:28.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:28.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:28.220 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:33:28.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:28.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:28.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:28.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[103001]: 6.1 scrub starts 2024-09-13T14:33:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3010128956' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[107556]: 6.1 scrub starts 2024-09-13T14:33:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3010128956' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:28 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:28 smithi189 ceph-mon[90992]: 6.1 scrub starts 2024-09-13T14:33:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3010128956' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:28 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:33:28.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:28.526 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:33:28.526 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:33:28.527 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2024-09-13T14:33:28.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:33:28.599 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:33:28.599 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:33:28.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:28.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:28.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:28.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:28.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:28.876 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:28.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:28.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:29.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:29.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:29.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:29.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:29.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:29.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:29.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:29.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:29.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:29 smithi189 ceph-mon[90992]: pgmap v579: 109 pgs: 5 peering, 5 unknown, 99 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:29.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:29 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:29.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:33:29.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:29 smithi189 ceph-mon[90992]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:33:29.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:29 smithi189 ceph-mon[90992]: 6.0 scrub starts 2024-09-13T14:33:29.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:29.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:29.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[103001]: pgmap v579: 109 pgs: 5 peering, 5 unknown, 99 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[103001]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[103001]: 6.0 scrub starts 2024-09-13T14:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[107556]: pgmap v579: 109 pgs: 5 peering, 5 unknown, 99 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[107556]: osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:29 smithi017 ceph-mon[107556]: 6.0 scrub starts 2024-09-13T14:33:29.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:29.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:29.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:29.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:29.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:29.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:30.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:30.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:30.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:30.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:30.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:30.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.14 scrub starts 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.14 scrub ok 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.3 scrub starts 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.6 scrub starts 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.0 scrub ok 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: 6.2 scrub starts 2024-09-13T14:33:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:33:30.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:30.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:30.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.14 scrub starts 2024-09-13T14:33:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.14 scrub ok 2024-09-13T14:33:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.3 scrub starts 2024-09-13T14:33:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.6 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.0 scrub ok 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: 6.2 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.14 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.14 scrub ok 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.3 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.6 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.0 scrub ok 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: 6.2 scrub starts 2024-09-13T14:33:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:33:30.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:33:30.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:30.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:30.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:30.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:30.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:31.036 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:31.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:31.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:31.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:31.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:31.307 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:31.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:31.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:31.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.4 scrub starts 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.3 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: pgmap v581: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.9 scrub starts 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.6 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.4 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.2 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.9 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.16 scrub starts 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.16 scrub ok 2024-09-13T14:33:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:31 smithi189 ceph-mon[90992]: 6.19 scrub starts 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.4 scrub starts 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.3 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: pgmap v581: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.9 scrub starts 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.6 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.4 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.2 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.9 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.16 scrub starts 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.16 scrub ok 2024-09-13T14:33:31.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[103001]: 6.19 scrub starts 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.4 scrub starts 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.3 scrub ok 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: pgmap v581: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.9 scrub starts 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.6 scrub ok 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.4 scrub ok 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.2 scrub ok 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.9 scrub ok 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.16 scrub starts 2024-09-13T14:33:31.538 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.16 scrub ok 2024-09-13T14:33:31.539 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:31 smithi017 ceph-mon[107556]: 6.19 scrub starts 2024-09-13T14:33:31.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:31.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:31.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:31.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:31.775 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:33:31.776 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:33:31.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:33:31.795 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:33:31.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:31.807 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:31.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:33:31.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:33:31.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:31.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:33:31.831 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:33:31.847 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:33:31.847 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:33:31.863 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:33:31.863 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:33:31.878 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:33:31.879 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:33:31.894 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:33:31.894 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:33:31.924 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:31.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:31.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:31.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:32.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:32.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:32.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:32.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:32.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:32.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:33:32.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:32.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:32.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:32.466 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: 6.11 scrub starts 2024-09-13T14:33:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: 6.11 scrub ok 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: 6.19 scrub ok 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: 6.5 scrub starts 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: 6.5 scrub ok 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3625104484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:33:32.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:33:32.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:32.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:32.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: 6.11 scrub starts 2024-09-13T14:33:32.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: 6.11 scrub ok 2024-09-13T14:33:32.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: 6.19 scrub ok 2024-09-13T14:33:32.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: 6.5 scrub starts 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: 6.5 scrub ok 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3625104484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: 6.11 scrub starts 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: 6.11 scrub ok 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: 6.19 scrub ok 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: 6.5 scrub starts 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: 6.5 scrub ok 2024-09-13T14:33:32.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3625104484' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:32.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:33:32.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:33:32.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:32.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:32.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:32.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:32.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:32.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:33.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:33.019 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:33:33.019 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:33:33.020 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:33:33.020 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:33:33.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:33.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:33.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:33.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:33.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:33.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: 6.1 scrub starts 2024-09-13T14:33:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: 6.1 scrub ok 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: pgmap v584: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 204 MiB/s wr, 5.91k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: 6.10 starting backfill to osd.5 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:33.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:33 smithi189 ceph-mon[90992]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:33:33.511 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:33.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:33.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:33.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: 6.1 scrub starts 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: 6.1 scrub ok 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: pgmap v584: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 204 MiB/s wr, 5.91k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: 6.10 starting backfill to osd.5 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[103001]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:33:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: 6.1 scrub starts 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: 6.1 scrub ok 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: pgmap v584: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 204 MiB/s wr, 5.91k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: 6.10 starting backfill to osd.1 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: 6.10 starting backfill to osd.5 from (0'0,0'0] MAX to 201'11727 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:33.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:33 smithi017 ceph-mon[107556]: osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:33:33.741 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:33.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:33.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:33.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:33.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:33.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:33.967 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:33.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:34.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:34.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:34.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:34.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:34.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:34.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:34.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:34.384 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:34 smithi189 ceph-mon[90992]: 6.8 scrub starts 2024-09-13T14:33:34.384 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:34 smithi189 ceph-mon[90992]: 6.8 scrub ok 2024-09-13T14:33:34.384 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:34 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:33:34.385 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:34 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:33:34.385 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:34 smithi189 ceph-mon[90992]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:33:34.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:34.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:34.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:34.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:34.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:34.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[103001]: 6.8 scrub starts 2024-09-13T14:33:34.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[103001]: 6.8 scrub ok 2024-09-13T14:33:34.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:33:34.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:33:34.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[103001]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:33:34.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[107556]: 6.8 scrub starts 2024-09-13T14:33:34.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[107556]: 6.8 scrub ok 2024-09-13T14:33:34.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:33:34.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:33:34.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:34 smithi017 ceph-mon[107556]: osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:33:34.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:33:34.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:34.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:34.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:34.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:35.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:35.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:35.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:35.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:35.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:35.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:35.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:35.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: pgmap v587: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/154044 objects misplaced (2.006%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: 6.13 scrub starts 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: 6.11 starting backfill to osd.3 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: 6.13 scrub ok 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[103001]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: pgmap v587: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/154044 objects misplaced (2.006%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: 6.13 scrub starts 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: 6.11 starting backfill to osd.3 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: 6.13 scrub ok 2024-09-13T14:33:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:35 smithi017 ceph-mon[107556]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:33:35.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:35.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:35.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: pgmap v587: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/154044 objects misplaced (2.006%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: 6.13 scrub starts 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: 6.11 starting backfill to osd.0 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: 6.11 starting backfill to osd.3 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: 6.11 starting backfill to osd.5 from (0'0,0'0] MAX to 202'12177 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: 6.13 scrub ok 2024-09-13T14:33:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:35 smithi189 ceph-mon[90992]: osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:33:35.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:35.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:35.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:35.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:35.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:35.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:35.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:35.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:35.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:35.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:36.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:36.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:36.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:36.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:36.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:36.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:36.370 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:33:36.370 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:33:36.370 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:33:36.371 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:33:36.371 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:33:36.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:36.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:33:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:36 smithi189 ceph-mon[90992]: 6.10 scrub starts 2024-09-13T14:33:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:33:36.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:36 smithi189 ceph-mon[90992]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:33:36.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:36.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:36.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:33:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[103001]: 6.10 scrub starts 2024-09-13T14:33:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:33:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[103001]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:33:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:33:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[107556]: 6.10 scrub starts 2024-09-13T14:33:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:33:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:36 smithi017 ceph-mon[107556]: osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:33:36.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:36.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:36.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:36.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:36.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:37.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:37.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:37.076 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:33:37.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:37.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:37.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:37.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:37.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:37.411 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:37.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:37] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.51.0" 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[103001]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/152793 objects misplaced (2.022%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[103001]: 6.15 scrub starts 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[103001]: 6.10 scrub ok 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[103001]: 6.15 scrub ok 2024-09-13T14:33:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[103001]: 6.12 starting backfill to osd.7 from (0'0,0'0] MAX to 204'12280 2024-09-13T14:33:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[107556]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/152793 objects misplaced (2.022%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[107556]: 6.15 scrub starts 2024-09-13T14:33:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[107556]: 6.10 scrub ok 2024-09-13T14:33:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[107556]: 6.15 scrub ok 2024-09-13T14:33:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:37 smithi017 ceph-mon[107556]: 6.12 starting backfill to osd.7 from (0'0,0'0] MAX to 204'12280 2024-09-13T14:33:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:37 smithi189 ceph-mon[90992]: pgmap v590: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3090/152793 objects misplaced (2.022%); 768 KiB/s, 11 objects/s recovering 2024-09-13T14:33:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:37 smithi189 ceph-mon[90992]: 6.15 scrub starts 2024-09-13T14:33:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:37 smithi189 ceph-mon[90992]: 6.10 scrub ok 2024-09-13T14:33:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:37 smithi189 ceph-mon[90992]: 6.15 scrub ok 2024-09-13T14:33:37.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:37 smithi189 ceph-mon[90992]: 6.12 starting backfill to osd.7 from (0'0,0'0] MAX to 204'12280 2024-09-13T14:33:37.522 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:33:37.524 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:33:37.547 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:37.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:37.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:37.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:37.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:37.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:37.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:37.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:37.954 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:37.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:38.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:38.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:38.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:38.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:38.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:38.344 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:38.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:38.446 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[103001]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:33:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[103001]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[103001]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:33:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[107556]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:33:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[107556]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:38 smithi017 ceph-mon[107556]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:33:38.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:38.576 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:38.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:38.677 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:38 smithi189 ceph-mon[90992]: osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:33:38.677 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:38 smithi189 ceph-mon[90992]: from='client.35532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:33:38.677 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:38 smithi189 ceph-mon[90992]: osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:33:38.704 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:38.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:38.825 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":592,"stamp":"2024-09-13T14:33:37.258923+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3335854053,"num_objects":50859,"num_object_clones":0,"num_object_copies":152577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4650,"num_objects_unfound":0,"num_objects_dirty":50859,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":131561,"num_write_kb":5245552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":213296,"ondisk_log_size":213296,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":328,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12316232,"kb_used_data":9798488,"kb_used_omap":35,"kb_used_meta":2517532,"kb_avail":737644984,"statfs":{"total":767960285184,"available":755348463616,"internally_reserved":0,"allocated":10033651712,"data_stored":10025277661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35847,"internal_metadata":2577953785},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-32047104,"num_objects":-489,"num_object_clones":0,"num_object_copies":-1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1560,"num_objects_unfound":0,"num_objects_dirty":-489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-11795,"num_write_kb":-497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.026312"},"pg_stats":[{"pgid":"6.16","version":"206'12089","reported_seq":12206,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.503168+0000","last_change":"2024-09-13T14:33:30.753654+0000","last_active":"2024-09-13T14:33:34.503168+0000","last_peered":"2024-09-13T14:33:34.503168+0000","last_clean":"2024-09-13T14:33:34.503168+0000","last_became_active":"2024-09-13T14:33:23.212048+0000","last_became_peered":"2024-09-13T14:33:23.212048+0000","last_unstale":"2024-09-13T14:33:34.503168+0000","last_undegraded":"2024-09-13T14:33:34.503168+0000","last_fullsized":"2024-09-13T14:33:34.503168+0000","mapping_epoch":199,"log_start":"183'5200","ondisk_log_start":"183'5200","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"202'12060","last_scrub_stamp":"2024-09-13T14:33:30.753288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:30.753288+0000","objects_scrubbed":1638,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:29:00.061300+0000","scrub_duration":332,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":280,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.246628+0000","last_change":"2024-09-13T14:24:00.661235+0000","last_active":"2024-09-13T14:33:34.246628+0000","last_peered":"2024-09-13T14:33:34.246628+0000","last_clean":"2024-09-13T14:33:34.246628+0000","last_became_active":"2024-09-13T14:24:00.660826+0000","last_became_peered":"2024-09-13T14:24:00.660826+0000","last_unstale":"2024-09-13T14:33:34.246628+0000","last_undegraded":"2024-09-13T14:33:34.246628+0000","last_fullsized":"2024-09-13T14:33:34.246628+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:04.670292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239523+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:33:31.239523+0000","last_peered":"2024-09-13T14:33:31.239523+0000","last_clean":"2024-09-13T14:33:31.239523+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:33:31.239523+0000","last_undegraded":"2024-09-13T14:33:31.239523+0000","last_fullsized":"2024-09-13T14:33:31.239523+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"204'12318","reported_seq":12367,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.926229+0000","last_change":"2024-09-13T14:33:23.222687+0000","last_active":"2024-09-13T14:33:32.926229+0000","last_peered":"2024-09-13T14:33:32.926229+0000","last_clean":"2024-09-13T14:33:32.926229+0000","last_became_active":"2024-09-13T14:33:23.199616+0000","last_became_peered":"2024-09-13T14:33:23.199616+0000","last_unstale":"2024-09-13T14:33:32.926229+0000","last_undegraded":"2024-09-13T14:33:32.926229+0000","last_fullsized":"2024-09-13T14:33:32.926229+0000","mapping_epoch":199,"log_start":"178'2000","ondisk_log_start":"178'2000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10318,"log_dups_size":0,"ondisk_log_size":10318,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105414656,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6299,"num_write_kb":253024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235577+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:33:31.235577+0000","last_peered":"2024-09-13T14:33:31.235577+0000","last_clean":"2024-09-13T14:33:31.235577+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:33:31.235577+0000","last_undegraded":"2024-09-13T14:33:31.235577+0000","last_fullsized":"2024-09-13T14:33:31.235577+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":272,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.240305+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:33:32.240305+0000","last_peered":"2024-09-13T14:33:32.240305+0000","last_clean":"2024-09-13T14:33:32.240305+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:33:32.240305+0000","last_undegraded":"2024-09-13T14:33:32.240305+0000","last_fullsized":"2024-09-13T14:33:32.240305+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"204'12335","reported_seq":12445,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.549007+0000","last_change":"2024-09-13T14:33:29.103003+0000","last_active":"2024-09-13T14:33:32.549007+0000","last_peered":"2024-09-13T14:33:32.549007+0000","last_clean":"2024-09-13T14:33:32.549007+0000","last_became_active":"2024-09-13T14:33:23.200340+0000","last_became_peered":"2024-09-13T14:33:23.200340+0000","last_unstale":"2024-09-13T14:33:32.549007+0000","last_undegraded":"2024-09-13T14:33:32.549007+0000","last_fullsized":"2024-09-13T14:33:32.549007+0000","mapping_epoch":199,"log_start":"181'4100","ondisk_log_start":"181'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"201'12271","last_scrub_stamp":"2024-09-13T14:33:29.102822+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:29.102822+0000","objects_scrubbed":1640,"log_size":8235,"log_dups_size":0,"ondisk_log_size":8235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:44.443909+0000","scrub_duration":634,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":68,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":272,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241527+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:33:32.241527+0000","last_peered":"2024-09-13T14:33:32.241527+0000","last_clean":"2024-09-13T14:33:32.241527+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:33:32.241527+0000","last_undegraded":"2024-09-13T14:33:32.241527+0000","last_fullsized":"2024-09-13T14:33:32.241527+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":151,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239293+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:33:31.239293+0000","last_peered":"2024-09-13T14:33:31.239293+0000","last_clean":"2024-09-13T14:33:31.239293+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:33:31.239293+0000","last_undegraded":"2024-09-13T14:33:31.239293+0000","last_fullsized":"2024-09-13T14:33:31.239293+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"204'11980","reported_seq":12029,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.823293+0000","last_change":"2024-09-13T14:33:23.227126+0000","last_active":"2024-09-13T14:33:32.823293+0000","last_peered":"2024-09-13T14:33:32.823293+0000","last_clean":"2024-09-13T14:33:32.823293+0000","last_became_active":"2024-09-13T14:33:23.202395+0000","last_became_peered":"2024-09-13T14:33:23.202395+0000","last_unstale":"2024-09-13T14:33:32.823293+0000","last_undegraded":"2024-09-13T14:33:32.823293+0000","last_fullsized":"2024-09-13T14:33:32.823293+0000","mapping_epoch":199,"log_start":"182'4100","ondisk_log_start":"182'4100","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7880,"log_dups_size":0,"ondisk_log_size":7880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6150,"num_write_kb":246720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":279,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.249917+0000","last_change":"2024-09-13T14:24:00.651299+0000","last_active":"2024-09-13T14:33:34.249917+0000","last_peered":"2024-09-13T14:33:34.249917+0000","last_clean":"2024-09-13T14:33:34.249917+0000","last_became_active":"2024-09-13T14:24:00.650974+0000","last_became_peered":"2024-09-13T14:24:00.650974+0000","last_unstale":"2024-09-13T14:33:34.249917+0000","last_undegraded":"2024-09-13T14:33:34.249917+0000","last_fullsized":"2024-09-13T14:33:34.249917+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:53.112544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":176,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242174+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:33:31.242174+0000","last_peered":"2024-09-13T14:33:31.242174+0000","last_clean":"2024-09-13T14:33:31.242174+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:33:31.242174+0000","last_undegraded":"2024-09-13T14:33:31.242174+0000","last_fullsized":"2024-09-13T14:33:31.242174+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"204'12206","reported_seq":12245,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.870029+0000","last_change":"2024-09-13T14:30:27.500678+0000","last_active":"2024-09-13T14:33:32.870029+0000","last_peered":"2024-09-13T14:33:32.870029+0000","last_clean":"2024-09-13T14:33:32.870029+0000","last_became_active":"2024-09-13T14:30:27.500235+0000","last_became_peered":"2024-09-13T14:30:27.500235+0000","last_unstale":"2024-09-13T14:33:32.870029+0000","last_undegraded":"2024-09-13T14:33:32.870029+0000","last_fullsized":"2024-09-13T14:33:32.870029+0000","mapping_epoch":173,"log_start":"179'3080","ondisk_log_start":"179'3080","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:11.965913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202375168,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":489408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":169,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239681+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:33:31.239681+0000","last_peered":"2024-09-13T14:33:31.239681+0000","last_clean":"2024-09-13T14:33:31.239681+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:33:31.239681+0000","last_undegraded":"2024-09-13T14:33:31.239681+0000","last_fullsized":"2024-09-13T14:33:31.239681+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":150,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.234918+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:33:31.234918+0000","last_peered":"2024-09-13T14:33:31.234918+0000","last_clean":"2024-09-13T14:33:31.234918+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:33:31.234918+0000","last_undegraded":"2024-09-13T14:33:31.234918+0000","last_fullsized":"2024-09-13T14:33:31.234918+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"206'12276","reported_seq":12359,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.479574+0000","last_change":"2024-09-13T14:30:27.502589+0000","last_active":"2024-09-13T14:33:34.479574+0000","last_peered":"2024-09-13T14:33:34.479574+0000","last_clean":"2024-09-13T14:33:34.479574+0000","last_became_active":"2024-09-13T14:30:27.502327+0000","last_became_peered":"2024-09-13T14:30:27.502327+0000","last_unstale":"2024-09-13T14:33:34.479574+0000","last_undegraded":"2024-09-13T14:33:34.479574+0000","last_fullsized":"2024-09-13T14:33:34.479574+0000","mapping_epoch":173,"log_start":"187'6666","ondisk_log_start":"187'6666","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:27.123011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206045184,"num_objects":3144,"num_object_clones":0,"num_object_copies":9432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12276,"num_write_kb":493440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239663+0000","last_change":"2024-09-13T14:22:54.185047+0000","last_active":"2024-09-13T14:33:31.239663+0000","last_peered":"2024-09-13T14:33:31.239663+0000","last_clean":"2024-09-13T14:33:31.239663+0000","last_became_active":"2024-09-13T14:22:54.184693+0000","last_became_peered":"2024-09-13T14:22:54.184693+0000","last_unstale":"2024-09-13T14:33:31.239663+0000","last_undegraded":"2024-09-13T14:33:31.239663+0000","last_fullsized":"2024-09-13T14:33:31.239663+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:54.325090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":185,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230672+0000","last_change":"2024-09-13T14:24:00.652215+0000","last_active":"2024-09-13T14:33:31.230672+0000","last_peered":"2024-09-13T14:33:31.230672+0000","last_clean":"2024-09-13T14:33:31.230672+0000","last_became_active":"2024-09-13T14:24:00.651926+0000","last_became_peered":"2024-09-13T14:24:00.651926+0000","last_unstale":"2024-09-13T14:33:31.230672+0000","last_undegraded":"2024-09-13T14:33:31.230672+0000","last_fullsized":"2024-09-13T14:33:31.230672+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:29.397573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"204'12189","reported_seq":12236,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.914564+0000","last_change":"2024-09-13T14:33:23.228693+0000","last_active":"2024-09-13T14:33:32.914564+0000","last_peered":"2024-09-13T14:33:32.914564+0000","last_clean":"2024-09-13T14:33:32.914564+0000","last_became_active":"2024-09-13T14:33:23.212194+0000","last_became_peered":"2024-09-13T14:33:23.212194+0000","last_unstale":"2024-09-13T14:33:32.914564+0000","last_undegraded":"2024-09-13T14:33:32.914564+0000","last_fullsized":"2024-09-13T14:33:32.914564+0000","mapping_epoch":173,"log_start":"182'4400","ondisk_log_start":"182'4400","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102334464,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6237,"num_write_kb":249568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":152,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239668+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:33:31.239668+0000","last_peered":"2024-09-13T14:33:31.239668+0000","last_clean":"2024-09-13T14:33:31.239668+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:33:31.239668+0000","last_undegraded":"2024-09-13T14:33:31.239668+0000","last_fullsized":"2024-09-13T14:33:31.239668+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":167,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242517+0000","last_change":"2024-09-13T14:22:54.185164+0000","last_active":"2024-09-13T14:33:31.242517+0000","last_peered":"2024-09-13T14:33:31.242517+0000","last_clean":"2024-09-13T14:33:31.242517+0000","last_became_active":"2024-09-13T14:22:54.184390+0000","last_became_peered":"2024-09-13T14:22:54.184390+0000","last_unstale":"2024-09-13T14:33:31.242517+0000","last_undegraded":"2024-09-13T14:33:31.242517+0000","last_fullsized":"2024-09-13T14:33:31.242517+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:02.824051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"204'12071","reported_seq":12182,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.580739+0000","last_change":"2024-09-13T14:33:30.375851+0000","last_active":"2024-09-13T14:33:32.580739+0000","last_peered":"2024-09-13T14:33:32.580739+0000","last_clean":"2024-09-13T14:33:32.580739+0000","last_became_active":"2024-09-13T14:33:23.218409+0000","last_became_peered":"2024-09-13T14:33:23.218409+0000","last_unstale":"2024-09-13T14:33:32.580739+0000","last_undegraded":"2024-09-13T14:33:32.580739+0000","last_fullsized":"2024-09-13T14:33:32.580739+0000","mapping_epoch":173,"log_start":"179'2700","ondisk_log_start":"179'2700","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"202'12048","last_scrub_stamp":"2024-09-13T14:33:30.375747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:30.375747+0000","objects_scrubbed":1610,"log_size":9371,"log_dups_size":0,"ondisk_log_size":9371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:12.062592+0000","scrub_duration":442,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103481344,"num_objects":1579,"num_object_clones":0,"num_object_copies":4737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235435+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:33:31.235435+0000","last_peered":"2024-09-13T14:33:31.235435+0000","last_clean":"2024-09-13T14:33:31.235435+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:33:31.235435+0000","last_undegraded":"2024-09-13T14:33:31.235435+0000","last_fullsized":"2024-09-13T14:33:31.235435+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":273,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241194+0000","last_change":"2024-09-13T14:22:54.183712+0000","last_active":"2024-09-13T14:33:32.241194+0000","last_peered":"2024-09-13T14:33:32.241194+0000","last_clean":"2024-09-13T14:33:32.241194+0000","last_became_active":"2024-09-13T14:22:54.183395+0000","last_became_peered":"2024-09-13T14:22:54.183395+0000","last_unstale":"2024-09-13T14:33:32.241194+0000","last_undegraded":"2024-09-13T14:33:32.241194+0000","last_fullsized":"2024-09-13T14:33:32.241194+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:22.563769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"204'12471","reported_seq":12510,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.965559+0000","last_change":"2024-09-13T14:30:27.501830+0000","last_active":"2024-09-13T14:33:32.965559+0000","last_peered":"2024-09-13T14:33:32.965559+0000","last_clean":"2024-09-13T14:33:32.965559+0000","last_became_active":"2024-09-13T14:30:27.501398+0000","last_became_peered":"2024-09-13T14:30:27.501398+0000","last_unstale":"2024-09-13T14:33:32.965559+0000","last_undegraded":"2024-09-13T14:33:32.965559+0000","last_fullsized":"2024-09-13T14:33:32.965559+0000","mapping_epoch":173,"log_start":"184'5635","ondisk_log_start":"184'5635","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:52.275184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12471,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":315,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239165+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:33:31.239165+0000","last_peered":"2024-09-13T14:33:31.239165+0000","last_clean":"2024-09-13T14:33:31.239165+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:33:31.239165+0000","last_undegraded":"2024-09-13T14:33:31.239165+0000","last_fullsized":"2024-09-13T14:33:31.239165+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":278,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244587+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:33:34.244587+0000","last_peered":"2024-09-13T14:33:34.244587+0000","last_clean":"2024-09-13T14:33:34.244587+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:33:34.244587+0000","last_undegraded":"2024-09-13T14:33:34.244587+0000","last_fullsized":"2024-09-13T14:33:34.244587+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.231053+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:33:31.231053+0000","last_peered":"2024-09-13T14:33:31.231053+0000","last_clean":"2024-09-13T14:33:31.231053+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:33:31.231053+0000","last_undegraded":"2024-09-13T14:33:31.231053+0000","last_fullsized":"2024-09-13T14:33:31.231053+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"206'12493","reported_seq":12581,"reported_epoch":208,"state":"active+clean","last_fresh":"2024-09-13T14:33:36.240990+0000","last_change":"2024-09-13T14:30:27.505193+0000","last_active":"2024-09-13T14:33:36.240990+0000","last_peered":"2024-09-13T14:33:36.240990+0000","last_clean":"2024-09-13T14:33:36.240990+0000","last_became_active":"2024-09-13T14:30:27.504909+0000","last_became_peered":"2024-09-13T14:30:27.504909+0000","last_unstale":"2024-09-13T14:33:36.240990+0000","last_undegraded":"2024-09-13T14:33:36.240990+0000","last_fullsized":"2024-09-13T14:33:36.240990+0000","mapping_epoch":173,"log_start":"182'4575","ondisk_log_start":"182'4575","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7918,"log_dups_size":0,"ondisk_log_size":7918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:52.846751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205193216,"num_objects":3131,"num_object_clones":0,"num_object_copies":9393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12493,"num_write_kb":499968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":62,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.234926+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:33:31.234926+0000","last_peered":"2024-09-13T14:33:31.234926+0000","last_clean":"2024-09-13T14:33:31.234926+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:33:31.234926+0000","last_undegraded":"2024-09-13T14:33:31.234926+0000","last_fullsized":"2024-09-13T14:33:31.234926+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242638+0000","last_change":"2024-09-13T14:22:54.185321+0000","last_active":"2024-09-13T14:33:31.242638+0000","last_peered":"2024-09-13T14:33:31.242638+0000","last_clean":"2024-09-13T14:33:31.242638+0000","last_became_active":"2024-09-13T14:22:54.184814+0000","last_became_peered":"2024-09-13T14:22:54.184814+0000","last_unstale":"2024-09-13T14:33:31.242638+0000","last_undegraded":"2024-09-13T14:33:31.242638+0000","last_fullsized":"2024-09-13T14:33:31.242638+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:11.564937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":178,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.231118+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:33:31.231118+0000","last_peered":"2024-09-13T14:33:31.231118+0000","last_clean":"2024-09-13T14:33:31.231118+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:33:31.231118+0000","last_undegraded":"2024-09-13T14:33:31.231118+0000","last_fullsized":"2024-09-13T14:33:31.231118+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"206'12451","reported_seq":12534,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.480120+0000","last_change":"2024-09-13T14:30:27.503642+0000","last_active":"2024-09-13T14:33:34.480120+0000","last_peered":"2024-09-13T14:33:34.480120+0000","last_clean":"2024-09-13T14:33:34.480120+0000","last_became_active":"2024-09-13T14:30:27.503347+0000","last_became_peered":"2024-09-13T14:30:27.503347+0000","last_unstale":"2024-09-13T14:33:34.480120+0000","last_undegraded":"2024-09-13T14:33:34.480120+0000","last_fullsized":"2024-09-13T14:33:34.480120+0000","mapping_epoch":173,"log_start":"187'6853","ondisk_log_start":"187'6853","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:02.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12451,"num_write_kb":498368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":169,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230781+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:33:31.230781+0000","last_peered":"2024-09-13T14:33:31.230781+0000","last_clean":"2024-09-13T14:33:31.230781+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:33:31.230781+0000","last_undegraded":"2024-09-13T14:33:31.230781+0000","last_fullsized":"2024-09-13T14:33:31.230781+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.231268+0000","last_change":"2024-09-13T14:22:54.187368+0000","last_active":"2024-09-13T14:33:31.231268+0000","last_peered":"2024-09-13T14:33:31.231268+0000","last_clean":"2024-09-13T14:33:31.231268+0000","last_became_active":"2024-09-13T14:22:54.186658+0000","last_became_peered":"2024-09-13T14:22:54.186658+0000","last_unstale":"2024-09-13T14:33:31.231268+0000","last_undegraded":"2024-09-13T14:33:31.231268+0000","last_fullsized":"2024-09-13T14:33:31.231268+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:31.148276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"204'12431","reported_seq":12471,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.882905+0000","last_change":"2024-09-13T14:30:27.499796+0000","last_active":"2024-09-13T14:33:32.882905+0000","last_peered":"2024-09-13T14:33:32.882905+0000","last_clean":"2024-09-13T14:33:32.882905+0000","last_became_active":"2024-09-13T14:30:27.499475+0000","last_became_peered":"2024-09-13T14:30:27.499475+0000","last_unstale":"2024-09-13T14:33:32.882905+0000","last_undegraded":"2024-09-13T14:33:32.882905+0000","last_fullsized":"2024-09-13T14:33:32.882905+0000","mapping_epoch":173,"log_start":"180'3297","ondisk_log_start":"180'3297","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9134,"log_dups_size":0,"ondisk_log_size":9134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:07.838538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205520920,"num_objects":3137,"num_object_clones":0,"num_object_copies":9411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3137,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12431,"num_write_kb":498113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":55,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235059+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:33:31.235059+0000","last_peered":"2024-09-13T14:33:31.235059+0000","last_clean":"2024-09-13T14:33:31.235059+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:33:31.235059+0000","last_undegraded":"2024-09-13T14:33:31.235059+0000","last_fullsized":"2024-09-13T14:33:31.235059+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":281,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244480+0000","last_change":"2024-09-13T14:24:00.652802+0000","last_active":"2024-09-13T14:33:34.244480+0000","last_peered":"2024-09-13T14:33:34.244480+0000","last_clean":"2024-09-13T14:33:34.244480+0000","last_became_active":"2024-09-13T14:24:00.652155+0000","last_became_peered":"2024-09-13T14:24:00.652155+0000","last_unstale":"2024-09-13T14:33:34.244480+0000","last_undegraded":"2024-09-13T14:33:34.244480+0000","last_fullsized":"2024-09-13T14:33:34.244480+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:41.873792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":262,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.239241+0000","last_change":"2024-09-13T14:24:00.654735+0000","last_active":"2024-09-13T14:33:32.239241+0000","last_peered":"2024-09-13T14:33:32.239241+0000","last_clean":"2024-09-13T14:33:32.239241+0000","last_became_active":"2024-09-13T14:24:00.654401+0000","last_became_peered":"2024-09-13T14:24:00.654401+0000","last_unstale":"2024-09-13T14:33:32.239241+0000","last_undegraded":"2024-09-13T14:33:32.239241+0000","last_fullsized":"2024-09-13T14:33:32.239241+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:05.461352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"204'12336","reported_seq":12386,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.825244+0000","last_change":"2024-09-13T14:33:23.199119+0000","last_active":"2024-09-13T14:33:32.825244+0000","last_peered":"2024-09-13T14:33:32.825244+0000","last_clean":"2024-09-13T14:33:32.825244+0000","last_became_active":"2024-09-13T14:33:23.174924+0000","last_became_peered":"2024-09-13T14:33:23.174924+0000","last_unstale":"2024-09-13T14:33:32.825244+0000","last_undegraded":"2024-09-13T14:33:32.825244+0000","last_fullsized":"2024-09-13T14:33:32.825244+0000","mapping_epoch":173,"log_start":"178'2000","ondisk_log_start":"178'2000","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":10336,"log_dups_size":0,"ondisk_log_size":10336,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104235008,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6316,"num_write_kb":253024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":306,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239242+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:33:31.239242+0000","last_peered":"2024-09-13T14:33:31.239242+0000","last_clean":"2024-09-13T14:33:31.239242+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:33:31.239242+0000","last_undegraded":"2024-09-13T14:33:31.239242+0000","last_fullsized":"2024-09-13T14:33:31.239242+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":769,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235106+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:33:31.235106+0000","last_peered":"2024-09-13T14:33:31.235106+0000","last_clean":"2024-09-13T14:33:31.235106+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:33:31.235106+0000","last_undegraded":"2024-09-13T14:33:31.235106+0000","last_fullsized":"2024-09-13T14:33:31.235106+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244516+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:33:34.244516+0000","last_peered":"2024-09-13T14:33:34.244516+0000","last_clean":"2024-09-13T14:33:34.244516+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:33:34.244516+0000","last_undegraded":"2024-09-13T14:33:34.244516+0000","last_fullsized":"2024-09-13T14:33:34.244516+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242458+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:33:31.242458+0000","last_peered":"2024-09-13T14:33:31.242458+0000","last_clean":"2024-09-13T14:33:31.242458+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:33:31.242458+0000","last_undegraded":"2024-09-13T14:33:31.242458+0000","last_fullsized":"2024-09-13T14:33:31.242458+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"204'12330","reported_seq":12442,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.462218+0000","last_change":"2024-09-13T14:33:30.030321+0000","last_active":"2024-09-13T14:33:32.462218+0000","last_peered":"2024-09-13T14:33:32.462218+0000","last_clean":"2024-09-13T14:33:32.462218+0000","last_became_active":"2024-09-13T14:33:23.185422+0000","last_became_peered":"2024-09-13T14:33:23.185422+0000","last_unstale":"2024-09-13T14:33:32.462218+0000","last_undegraded":"2024-09-13T14:33:32.462218+0000","last_fullsized":"2024-09-13T14:33:32.462218+0000","mapping_epoch":173,"log_start":"181'4100","ondisk_log_start":"181'4100","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'12294","last_scrub_stamp":"2024-09-13T14:33:30.030016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:30.030016+0000","objects_scrubbed":1597,"log_size":8230,"log_dups_size":0,"ondisk_log_size":8230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:23:24.225456+0000","scrub_duration":516,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101842944,"num_objects":1554,"num_object_clones":0,"num_object_copies":4662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":107,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238517+0000","last_change":"2024-09-13T14:30:39.822827+0000","last_active":"2024-09-13T14:33:32.238517+0000","last_peered":"2024-09-13T14:33:32.238517+0000","last_clean":"2024-09-13T14:33:32.238517+0000","last_became_active":"2024-09-13T14:30:11.437578+0000","last_became_peered":"2024-09-13T14:30:11.437578+0000","last_unstale":"2024-09-13T14:33:32.238517+0000","last_undegraded":"2024-09-13T14:33:32.238517+0000","last_fullsized":"2024-09-13T14:33:32.238517+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.822747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.822747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:44.769254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":577,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239144+0000","last_change":"2024-09-13T14:22:54.184048+0000","last_active":"2024-09-13T14:33:31.239144+0000","last_peered":"2024-09-13T14:33:31.239144+0000","last_clean":"2024-09-13T14:33:31.239144+0000","last_became_active":"2024-09-13T14:22:54.183536+0000","last_became_peered":"2024-09-13T14:22:54.183536+0000","last_unstale":"2024-09-13T14:33:31.239144+0000","last_undegraded":"2024-09-13T14:33:31.239144+0000","last_fullsized":"2024-09-13T14:33:31.239144+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:09.965043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":272,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.239932+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:33:32.239932+0000","last_peered":"2024-09-13T14:33:32.239932+0000","last_clean":"2024-09-13T14:33:32.239932+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:33:32.239932+0000","last_undegraded":"2024-09-13T14:33:32.239932+0000","last_fullsized":"2024-09-13T14:33:32.239932+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":284,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241592+0000","last_change":"2024-09-13T14:23:27.719308+0000","last_active":"2024-09-13T14:33:32.241592+0000","last_peered":"2024-09-13T14:33:32.241592+0000","last_clean":"2024-09-13T14:33:32.241592+0000","last_became_active":"2024-09-13T14:23:27.719008+0000","last_became_peered":"2024-09-13T14:23:27.719008+0000","last_unstale":"2024-09-13T14:33:32.241592+0000","last_undegraded":"2024-09-13T14:33:32.241592+0000","last_fullsized":"2024-09-13T14:33:32.241592+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:41.222603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"204'11944","reported_seq":12068,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.918202+0000","last_change":"2024-09-13T14:33:31.290560+0000","last_active":"2024-09-13T14:33:32.918202+0000","last_peered":"2024-09-13T14:33:32.918202+0000","last_clean":"2024-09-13T14:33:32.918202+0000","last_became_active":"2024-09-13T14:33:23.174320+0000","last_became_peered":"2024-09-13T14:33:23.174320+0000","last_unstale":"2024-09-13T14:33:32.918202+0000","last_undegraded":"2024-09-13T14:33:32.918202+0000","last_fullsized":"2024-09-13T14:33:32.918202+0000","mapping_epoch":173,"log_start":"182'4100","ondisk_log_start":"182'4100","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"202'11927","last_scrub_stamp":"2024-09-13T14:33:31.290494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:31.290494+0000","objects_scrubbed":1607,"log_size":7844,"log_dups_size":0,"ondisk_log_size":7844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:01.007794+0000","scrub_duration":0.14807429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":315,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239237+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:33:31.239237+0000","last_peered":"2024-09-13T14:33:31.239237+0000","last_clean":"2024-09-13T14:33:31.239237+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:33:31.239237+0000","last_undegraded":"2024-09-13T14:33:31.239237+0000","last_fullsized":"2024-09-13T14:33:31.239237+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":709,"reported_epoch":208,"state":"active+clean","last_fresh":"2024-09-13T14:33:36.325359+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:33:36.325359+0000","last_peered":"2024-09-13T14:33:36.325359+0000","last_clean":"2024-09-13T14:33:36.325359+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:33:36.325359+0000","last_undegraded":"2024-09-13T14:33:36.325359+0000","last_fullsized":"2024-09-13T14:33:36.325359+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":263,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.237919+0000","last_change":"2024-09-13T14:23:27.721221+0000","last_active":"2024-09-13T14:33:32.237919+0000","last_peered":"2024-09-13T14:33:32.237919+0000","last_clean":"2024-09-13T14:33:32.237919+0000","last_became_active":"2024-09-13T14:23:27.720647+0000","last_became_peered":"2024-09-13T14:23:27.720647+0000","last_unstale":"2024-09-13T14:33:32.237919+0000","last_undegraded":"2024-09-13T14:33:32.237919+0000","last_fullsized":"2024-09-13T14:33:32.237919+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:57.117177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":286,"reported_epoch":208,"state":"active+clean","last_fresh":"2024-09-13T14:33:36.326670+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:33:36.326670+0000","last_peered":"2024-09-13T14:33:36.326670+0000","last_clean":"2024-09-13T14:33:36.326670+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:33:36.326670+0000","last_undegraded":"2024-09-13T14:33:36.326670+0000","last_fullsized":"2024-09-13T14:33:36.326670+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"206'12090","reported_seq":12208,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.509326+0000","last_change":"2024-09-13T14:33:29.957715+0000","last_active":"2024-09-13T14:33:34.509326+0000","last_peered":"2024-09-13T14:33:34.509326+0000","last_clean":"2024-09-13T14:33:34.509326+0000","last_became_active":"2024-09-13T14:33:23.193012+0000","last_became_peered":"2024-09-13T14:33:23.193012+0000","last_unstale":"2024-09-13T14:33:34.509326+0000","last_undegraded":"2024-09-13T14:33:34.509326+0000","last_fullsized":"2024-09-13T14:33:34.509326+0000","mapping_epoch":173,"log_start":"183'5200","ondisk_log_start":"183'5200","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'12047","last_scrub_stamp":"2024-09-13T14:33:29.957620+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:29.957620+0000","objects_scrubbed":1610,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:23.180422+0000","scrub_duration":528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102170624,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":55,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.234955+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:33:31.234955+0000","last_peered":"2024-09-13T14:33:31.234955+0000","last_clean":"2024-09-13T14:33:31.234955+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:33:31.234955+0000","last_undegraded":"2024-09-13T14:33:31.234955+0000","last_fullsized":"2024-09-13T14:33:31.234955+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":818,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235210+0000","last_change":"2024-09-13T14:22:54.188799+0000","last_active":"2024-09-13T14:33:31.235210+0000","last_peered":"2024-09-13T14:33:31.235210+0000","last_clean":"2024-09-13T14:33:31.235210+0000","last_became_active":"2024-09-13T14:22:54.188284+0000","last_became_peered":"2024-09-13T14:22:54.188284+0000","last_unstale":"2024-09-13T14:33:31.235210+0000","last_undegraded":"2024-09-13T14:33:31.235210+0000","last_fullsized":"2024-09-13T14:33:31.235210+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:08:31.021446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":169,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230943+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:33:31.230943+0000","last_peered":"2024-09-13T14:33:31.230943+0000","last_clean":"2024-09-13T14:33:31.230943+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:33:31.230943+0000","last_undegraded":"2024-09-13T14:33:31.230943+0000","last_fullsized":"2024-09-13T14:33:31.230943+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238308+0000","last_change":"2024-09-13T14:24:00.655386+0000","last_active":"2024-09-13T14:33:32.238308+0000","last_peered":"2024-09-13T14:33:32.238308+0000","last_clean":"2024-09-13T14:33:32.238308+0000","last_became_active":"2024-09-13T14:24:00.654949+0000","last_became_peered":"2024-09-13T14:24:00.654949+0000","last_unstale":"2024-09-13T14:33:32.238308+0000","last_undegraded":"2024-09-13T14:33:32.238308+0000","last_fullsized":"2024-09-13T14:33:32.238308+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:34.026971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"204'11768","reported_seq":11881,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.561215+0000","last_change":"2024-09-13T14:33:29.542173+0000","last_active":"2024-09-13T14:33:32.561215+0000","last_peered":"2024-09-13T14:33:32.561215+0000","last_clean":"2024-09-13T14:33:32.561215+0000","last_became_active":"2024-09-13T14:33:23.216437+0000","last_became_peered":"2024-09-13T14:33:23.216437+0000","last_unstale":"2024-09-13T14:33:32.561215+0000","last_undegraded":"2024-09-13T14:33:32.561215+0000","last_fullsized":"2024-09-13T14:33:32.561215+0000","mapping_epoch":173,"log_start":"178'2400","ondisk_log_start":"178'2400","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'11717","last_scrub_stamp":"2024-09-13T14:33:29.542085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:29.542085+0000","objects_scrubbed":1662,"log_size":9368,"log_dups_size":0,"ondisk_log_size":9368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:42.397635+0000","scrub_duration":633,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105250816,"num_objects":1606,"num_object_clones":0,"num_object_copies":4818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":56,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":203,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242424+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:33:31.242424+0000","last_peered":"2024-09-13T14:33:31.242424+0000","last_clean":"2024-09-13T14:33:31.242424+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:33:31.242424+0000","last_undegraded":"2024-09-13T14:33:31.242424+0000","last_fullsized":"2024-09-13T14:33:31.242424+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235655+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:33:31.235655+0000","last_peered":"2024-09-13T14:33:31.235655+0000","last_clean":"2024-09-13T14:33:31.235655+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:33:31.235655+0000","last_undegraded":"2024-09-13T14:33:31.235655+0000","last_fullsized":"2024-09-13T14:33:31.235655+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":873,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238490+0000","last_change":"2024-09-13T14:22:54.183620+0000","last_active":"2024-09-13T14:33:32.238490+0000","last_peered":"2024-09-13T14:33:32.238490+0000","last_clean":"2024-09-13T14:33:32.238490+0000","last_became_active":"2024-09-13T14:22:54.182949+0000","last_became_peered":"2024-09-13T14:22:54.182949+0000","last_unstale":"2024-09-13T14:33:32.238490+0000","last_undegraded":"2024-09-13T14:33:32.238490+0000","last_fullsized":"2024-09-13T14:33:32.238490+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:22.622393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":252,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238202+0000","last_change":"2024-09-13T14:22:54.188314+0000","last_active":"2024-09-13T14:33:32.238202+0000","last_peered":"2024-09-13T14:33:32.238202+0000","last_clean":"2024-09-13T14:33:32.238202+0000","last_became_active":"2024-09-13T14:22:54.187819+0000","last_became_peered":"2024-09-13T14:22:54.187819+0000","last_unstale":"2024-09-13T14:33:32.238202+0000","last_undegraded":"2024-09-13T14:33:32.238202+0000","last_fullsized":"2024-09-13T14:33:32.238202+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:08.928999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"204'12164","reported_seq":12277,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.543558+0000","last_change":"2024-09-13T14:33:31.749613+0000","last_active":"2024-09-13T14:33:32.543558+0000","last_peered":"2024-09-13T14:33:32.543558+0000","last_clean":"2024-09-13T14:33:32.543558+0000","last_became_active":"2024-09-13T14:33:23.230543+0000","last_became_peered":"2024-09-13T14:33:23.230543+0000","last_unstale":"2024-09-13T14:33:32.543558+0000","last_undegraded":"2024-09-13T14:33:32.543558+0000","last_fullsized":"2024-09-13T14:33:32.543558+0000","mapping_epoch":173,"log_start":"181'4000","ondisk_log_start":"181'4000","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"202'12156","last_scrub_stamp":"2024-09-13T14:33:31.749503+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:31.749503+0000","objects_scrubbed":1677,"log_size":8164,"log_dups_size":0,"ondisk_log_size":8164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:23.251068+0000","scrub_duration":167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109379584,"num_objects":1669,"num_object_clones":0,"num_object_copies":5007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":203,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242418+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:33:31.242418+0000","last_peered":"2024-09-13T14:33:31.242418+0000","last_clean":"2024-09-13T14:33:31.242418+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:33:31.242418+0000","last_undegraded":"2024-09-13T14:33:31.242418+0000","last_fullsized":"2024-09-13T14:33:31.242418+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238396+0000","last_change":"2024-09-13T14:23:27.720542+0000","last_active":"2024-09-13T14:33:32.238396+0000","last_peered":"2024-09-13T14:33:32.238396+0000","last_clean":"2024-09-13T14:33:32.238396+0000","last_became_active":"2024-09-13T14:23:27.720123+0000","last_became_peered":"2024-09-13T14:23:27.720123+0000","last_unstale":"2024-09-13T14:33:32.238396+0000","last_undegraded":"2024-09-13T14:33:32.238396+0000","last_fullsized":"2024-09-13T14:33:32.238396+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:29.536376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":514,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239659+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:33:31.239659+0000","last_peered":"2024-09-13T14:33:31.239659+0000","last_clean":"2024-09-13T14:33:31.239659+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:33:31.239659+0000","last_undegraded":"2024-09-13T14:33:31.239659+0000","last_fullsized":"2024-09-13T14:33:31.239659+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":262,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.240524+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:33:32.240524+0000","last_peered":"2024-09-13T14:33:32.240524+0000","last_clean":"2024-09-13T14:33:32.240524+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:33:32.240524+0000","last_undegraded":"2024-09-13T14:33:32.240524+0000","last_fullsized":"2024-09-13T14:33:32.240524+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"204'12233","reported_seq":12358,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.678332+0000","last_change":"2024-09-13T14:33:30.098033+0000","last_active":"2024-09-13T14:33:32.678332+0000","last_peered":"2024-09-13T14:33:32.678332+0000","last_clean":"2024-09-13T14:33:32.678332+0000","last_became_active":"2024-09-13T14:33:23.219889+0000","last_became_peered":"2024-09-13T14:33:23.219889+0000","last_unstale":"2024-09-13T14:33:32.678332+0000","last_undegraded":"2024-09-13T14:33:32.678332+0000","last_fullsized":"2024-09-13T14:33:32.678332+0000","mapping_epoch":173,"log_start":"182'4600","ondisk_log_start":"182'4600","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'12205","last_scrub_stamp":"2024-09-13T14:33:30.097937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:30.097937+0000","objects_scrubbed":1627,"log_size":7633,"log_dups_size":0,"ondisk_log_size":7633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:49.136376+0000","scrub_duration":0.48825949400000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104071168,"num_objects":1588,"num_object_clones":0,"num_object_copies":4764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":55,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230969+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:33:31.230969+0000","last_peered":"2024-09-13T14:33:31.230969+0000","last_clean":"2024-09-13T14:33:31.230969+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:33:31.230969+0000","last_undegraded":"2024-09-13T14:33:31.230969+0000","last_fullsized":"2024-09-13T14:33:31.230969+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":278,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.250214+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:33:34.250214+0000","last_peered":"2024-09-13T14:33:34.250214+0000","last_clean":"2024-09-13T14:33:34.250214+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:33:34.250214+0000","last_undegraded":"2024-09-13T14:33:34.250214+0000","last_fullsized":"2024-09-13T14:33:34.250214+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":606,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.239030+0000","last_change":"2024-09-13T14:22:54.185420+0000","last_active":"2024-09-13T14:33:32.239030+0000","last_peered":"2024-09-13T14:33:32.239030+0000","last_clean":"2024-09-13T14:33:32.239030+0000","last_became_active":"2024-09-13T14:22:54.184959+0000","last_became_peered":"2024-09-13T14:22:54.184959+0000","last_unstale":"2024-09-13T14:33:32.239030+0000","last_undegraded":"2024-09-13T14:33:32.239030+0000","last_fullsized":"2024-09-13T14:33:32.239030+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:19:11.848885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":275,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244212+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:33:34.244212+0000","last_peered":"2024-09-13T14:33:34.244212+0000","last_clean":"2024-09-13T14:33:34.244212+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:33:34.244212+0000","last_undegraded":"2024-09-13T14:33:34.244212+0000","last_fullsized":"2024-09-13T14:33:34.244212+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"204'12274","reported_seq":12398,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.909069+0000","last_change":"2024-09-13T14:33:29.637138+0000","last_active":"2024-09-13T14:33:32.909069+0000","last_peered":"2024-09-13T14:33:32.909069+0000","last_clean":"2024-09-13T14:33:32.909069+0000","last_became_active":"2024-09-13T14:33:23.204179+0000","last_became_peered":"2024-09-13T14:33:23.204179+0000","last_unstale":"2024-09-13T14:33:32.909069+0000","last_undegraded":"2024-09-13T14:33:32.909069+0000","last_fullsized":"2024-09-13T14:33:32.909069+0000","mapping_epoch":173,"log_start":"182'4400","ondisk_log_start":"182'4400","created":173,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"201'12220","last_scrub_stamp":"2024-09-13T14:33:29.637045+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:29.637045+0000","objects_scrubbed":1693,"log_size":7874,"log_dups_size":0,"ondisk_log_size":7874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:44.119962+0000","scrub_duration":0.48059620400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":116,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244773+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:33:34.244773+0000","last_peered":"2024-09-13T14:33:34.244773+0000","last_clean":"2024-09-13T14:33:34.244773+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:33:34.244773+0000","last_undegraded":"2024-09-13T14:33:34.244773+0000","last_fullsized":"2024-09-13T14:33:34.244773+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":276,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241411+0000","last_change":"2024-09-13T14:22:54.185199+0000","last_active":"2024-09-13T14:33:32.241411+0000","last_peered":"2024-09-13T14:33:32.241411+0000","last_clean":"2024-09-13T14:33:32.241411+0000","last_became_active":"2024-09-13T14:22:54.184873+0000","last_became_peered":"2024-09-13T14:22:54.184873+0000","last_unstale":"2024-09-13T14:33:32.241411+0000","last_undegraded":"2024-09-13T14:33:32.241411+0000","last_fullsized":"2024-09-13T14:33:32.241411+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:32.122605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":622,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242599+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:33:31.242599+0000","last_peered":"2024-09-13T14:33:31.242599+0000","last_clean":"2024-09-13T14:33:31.242599+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:33:31.242599+0000","last_undegraded":"2024-09-13T14:33:31.242599+0000","last_fullsized":"2024-09-13T14:33:31.242599+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230594+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:33:31.230594+0000","last_peered":"2024-09-13T14:33:31.230594+0000","last_clean":"2024-09-13T14:33:31.230594+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:33:31.230594+0000","last_undegraded":"2024-09-13T14:33:31.230594+0000","last_fullsized":"2024-09-13T14:33:31.230594+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.11","version":"200'12058","reported_seq":12133,"reported_epoch":200,"state":"active+clean","last_fresh":"2024-09-13T14:33:27.561963+0000","last_change":"2024-09-13T14:33:25.492379+0000","last_active":"2024-09-13T14:33:27.561963+0000","last_peered":"2024-09-13T14:33:27.561963+0000","last_clean":"2024-09-13T14:33:27.561963+0000","last_became_active":"2024-09-13T14:33:23.252042+0000","last_became_peered":"2024-09-13T14:33:23.252042+0000","last_unstale":"2024-09-13T14:33:27.561963+0000","last_undegraded":"2024-09-13T14:33:27.561963+0000","last_fullsized":"2024-09-13T14:33:27.561963+0000","mapping_epoch":199,"log_start":"181'4000","ondisk_log_start":"181'4000","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":8058,"log_dups_size":0,"ondisk_log_size":8058,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T14:33:30.468191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115572736,"num_objects":1764,"num_object_clones":0,"num_object_copies":5292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6109,"num_write_kb":251904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230852+0000","last_change":"2024-09-13T14:22:54.186355+0000","last_active":"2024-09-13T14:33:31.230852+0000","last_peered":"2024-09-13T14:33:31.230852+0000","last_clean":"2024-09-13T14:33:31.230852+0000","last_became_active":"2024-09-13T14:22:54.186040+0000","last_became_peered":"2024-09-13T14:22:54.186040+0000","last_unstale":"2024-09-13T14:33:31.230852+0000","last_undegraded":"2024-09-13T14:33:31.230852+0000","last_fullsized":"2024-09-13T14:33:31.230852+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:36.526907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":174,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.231131+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:33:31.231131+0000","last_peered":"2024-09-13T14:33:31.231131+0000","last_clean":"2024-09-13T14:33:31.231131+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:33:31.231131+0000","last_undegraded":"2024-09-13T14:33:31.231131+0000","last_fullsized":"2024-09-13T14:33:31.231131+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.10","version":"204'11747","reported_seq":11984,"reported_epoch":204,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:33:32.635543+0000","last_change":"2024-09-13T14:33:32.330743+0000","last_active":"2024-09-13T14:33:32.635543+0000","last_peered":"2024-09-13T14:33:32.635543+0000","last_clean":"2024-09-13T14:33:30.238669+0000","last_became_active":"2024-09-13T14:33:32.301979+0000","last_became_peered":"2024-09-13T14:33:32.301979+0000","last_unstale":"2024-09-13T14:33:32.635543+0000","last_undegraded":"2024-09-13T14:33:32.635543+0000","last_fullsized":"2024-09-13T14:33:32.635543+0000","mapping_epoch":203,"log_start":"178'2400","ondisk_log_start":"178'2400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":9347,"log_dups_size":0,"ondisk_log_size":9347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104300544,"num_objects":1592,"num_object_clones":0,"num_object_copies":4776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3090,"num_objects_unfound":0,"num_objects_dirty":1592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5996,"num_write_kb":242784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,2,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1592}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":262,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238051+0000","last_change":"2024-09-13T14:23:27.721461+0000","last_active":"2024-09-13T14:33:32.238051+0000","last_peered":"2024-09-13T14:33:32.238051+0000","last_clean":"2024-09-13T14:33:32.238051+0000","last_became_active":"2024-09-13T14:23:27.721088+0000","last_became_peered":"2024-09-13T14:23:27.721088+0000","last_unstale":"2024-09-13T14:33:32.238051+0000","last_undegraded":"2024-09-13T14:33:32.238051+0000","last_fullsized":"2024-09-13T14:33:32.238051+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:28.063653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":171,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.239556+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:33:31.239556+0000","last_peered":"2024-09-13T14:33:31.239556+0000","last_clean":"2024-09-13T14:33:31.239556+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:33:31.239556+0000","last_undegraded":"2024-09-13T14:33:31.239556+0000","last_fullsized":"2024-09-13T14:33:31.239556+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.13","version":"204'12244","reported_seq":12290,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.782788+0000","last_change":"2024-09-13T14:33:23.253477+0000","last_active":"2024-09-13T14:33:32.782788+0000","last_peered":"2024-09-13T14:33:32.782788+0000","last_clean":"2024-09-13T14:33:32.782788+0000","last_became_active":"2024-09-13T14:33:23.233313+0000","last_became_peered":"2024-09-13T14:33:23.233313+0000","last_unstale":"2024-09-13T14:33:32.782788+0000","last_undegraded":"2024-09-13T14:33:32.782788+0000","last_fullsized":"2024-09-13T14:33:32.782788+0000","mapping_epoch":199,"log_start":"182'4400","ondisk_log_start":"182'4400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7844,"log_dups_size":0,"ondisk_log_size":7844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108167168,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6251,"num_write_kb":252832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235309+0000","last_change":"2024-09-13T14:22:54.190139+0000","last_active":"2024-09-13T14:33:31.235309+0000","last_peered":"2024-09-13T14:33:31.235309+0000","last_clean":"2024-09-13T14:33:31.235309+0000","last_became_active":"2024-09-13T14:22:54.189840+0000","last_became_peered":"2024-09-13T14:22:54.189840+0000","last_unstale":"2024-09-13T14:33:31.235309+0000","last_undegraded":"2024-09-13T14:33:31.235309+0000","last_fullsized":"2024-09-13T14:33:31.235309+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:33.243402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":235,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244382+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:33:34.244382+0000","last_peered":"2024-09-13T14:33:34.244382+0000","last_clean":"2024-09-13T14:33:34.244382+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:33:34.244382+0000","last_undegraded":"2024-09-13T14:33:34.244382+0000","last_fullsized":"2024-09-13T14:33:34.244382+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.12","version":"204'12280","reported_seq":12334,"reported_epoch":208,"state":"activating+remapped","last_fresh":"2024-09-13T14:33:36.333662+0000","last_change":"2024-09-13T14:33:36.331535+0000","last_active":"2024-09-13T14:33:35.240211+0000","last_peered":"2024-09-13T14:33:33.003001+0000","last_clean":"2024-09-13T14:33:33.003001+0000","last_became_active":"2024-09-13T14:33:23.235056+0000","last_became_peered":"2024-09-13T14:33:23.235056+0000","last_unstale":"2024-09-13T14:33:36.333662+0000","last_undegraded":"2024-09-13T14:33:36.333662+0000","last_fullsized":"2024-09-13T14:33:36.333662+0000","mapping_epoch":207,"log_start":"182'4600","ondisk_log_start":"182'4600","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7680,"log_dups_size":0,"ondisk_log_size":7680,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102203392,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1560,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6298,"num_write_kb":251424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[2,4,3],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1560}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":271,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241327+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:33:32.241327+0000","last_peered":"2024-09-13T14:33:32.241327+0000","last_clean":"2024-09-13T14:33:32.241327+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:33:32.241327+0000","last_undegraded":"2024-09-13T14:33:32.241327+0000","last_fullsized":"2024-09-13T14:33:32.241327+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":280,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244136+0000","last_change":"2024-09-13T14:24:00.652920+0000","last_active":"2024-09-13T14:33:34.244136+0000","last_peered":"2024-09-13T14:33:34.244136+0000","last_clean":"2024-09-13T14:33:34.244136+0000","last_became_active":"2024-09-13T14:24:00.652579+0000","last_became_peered":"2024-09-13T14:24:00.652579+0000","last_unstale":"2024-09-13T14:33:34.244136+0000","last_undegraded":"2024-09-13T14:33:34.244136+0000","last_fullsized":"2024-09-13T14:33:34.244136+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:40.227874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":167,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.242514+0000","last_change":"2024-09-13T14:22:54.186024+0000","last_active":"2024-09-13T14:33:31.242514+0000","last_peered":"2024-09-13T14:33:31.242514+0000","last_clean":"2024-09-13T14:33:31.242514+0000","last_became_active":"2024-09-13T14:22:54.185658+0000","last_became_peered":"2024-09-13T14:22:54.185658+0000","last_unstale":"2024-09-13T14:33:31.242514+0000","last_undegraded":"2024-09-13T14:33:31.242514+0000","last_fullsized":"2024-09-13T14:33:31.242514+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:00.281851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":297,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244363+0000","last_change":"2024-09-13T14:24:00.652187+0000","last_active":"2024-09-13T14:33:34.244363+0000","last_peered":"2024-09-13T14:33:34.244363+0000","last_clean":"2024-09-13T14:33:34.244363+0000","last_became_active":"2024-09-13T14:24:00.651682+0000","last_became_peered":"2024-09-13T14:24:00.651682+0000","last_unstale":"2024-09-13T14:33:34.244363+0000","last_undegraded":"2024-09-13T14:33:34.244363+0000","last_fullsized":"2024-09-13T14:33:34.244363+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:11:15.089395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":273,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241534+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:33:32.241534+0000","last_peered":"2024-09-13T14:33:32.241534+0000","last_clean":"2024-09-13T14:33:32.241534+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:33:32.241534+0000","last_undegraded":"2024-09-13T14:33:32.241534+0000","last_fullsized":"2024-09-13T14:33:32.241534+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":263,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.238827+0000","last_change":"2024-09-13T14:24:00.654154+0000","last_active":"2024-09-13T14:33:32.238827+0000","last_peered":"2024-09-13T14:33:32.238827+0000","last_clean":"2024-09-13T14:33:32.238827+0000","last_became_active":"2024-09-13T14:24:00.653833+0000","last_became_peered":"2024-09-13T14:24:00.653833+0000","last_unstale":"2024-09-13T14:33:32.238827+0000","last_undegraded":"2024-09-13T14:33:32.238827+0000","last_fullsized":"2024-09-13T14:33:32.238827+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:09.948339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":279,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244828+0000","last_change":"2024-09-13T14:24:00.653872+0000","last_active":"2024-09-13T14:33:34.244828+0000","last_peered":"2024-09-13T14:33:34.244828+0000","last_clean":"2024-09-13T14:33:34.244828+0000","last_became_active":"2024-09-13T14:24:00.650820+0000","last_became_peered":"2024-09-13T14:24:00.650820+0000","last_unstale":"2024-09-13T14:33:34.244828+0000","last_undegraded":"2024-09-13T14:33:34.244828+0000","last_fullsized":"2024-09-13T14:33:34.244828+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:34.181662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":187,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230941+0000","last_change":"2024-09-13T14:23:27.719206+0000","last_active":"2024-09-13T14:33:31.230941+0000","last_peered":"2024-09-13T14:33:31.230941+0000","last_clean":"2024-09-13T14:33:31.230941+0000","last_became_active":"2024-09-13T14:23:27.718845+0000","last_became_peered":"2024-09-13T14:23:27.718845+0000","last_unstale":"2024-09-13T14:33:31.230941+0000","last_undegraded":"2024-09-13T14:33:31.230941+0000","last_fullsized":"2024-09-13T14:33:31.230941+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:49.589403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":304,"reported_epoch":208,"state":"active+clean","last_fresh":"2024-09-13T14:33:36.325242+0000","last_change":"2024-09-13T14:24:00.654663+0000","last_active":"2024-09-13T14:33:36.325242+0000","last_peered":"2024-09-13T14:33:36.325242+0000","last_clean":"2024-09-13T14:33:36.325242+0000","last_became_active":"2024-09-13T14:24:00.654370+0000","last_became_peered":"2024-09-13T14:24:00.654370+0000","last_unstale":"2024-09-13T14:33:36.325242+0000","last_undegraded":"2024-09-13T14:33:36.325242+0000","last_fullsized":"2024-09-13T14:33:36.325242+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:19.621541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":264,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.240493+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:33:32.240493+0000","last_peered":"2024-09-13T14:33:32.240493+0000","last_clean":"2024-09-13T14:33:32.240493+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:33:32.240493+0000","last_undegraded":"2024-09-13T14:33:32.240493+0000","last_fullsized":"2024-09-13T14:33:32.240493+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"204'12088","reported_seq":12199,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.598196+0000","last_change":"2024-09-13T14:33:31.043096+0000","last_active":"2024-09-13T14:33:32.598196+0000","last_peered":"2024-09-13T14:33:32.598196+0000","last_clean":"2024-09-13T14:33:32.598196+0000","last_became_active":"2024-09-13T14:33:23.258361+0000","last_became_peered":"2024-09-13T14:33:23.258361+0000","last_unstale":"2024-09-13T14:33:32.598196+0000","last_undegraded":"2024-09-13T14:33:32.598196+0000","last_fullsized":"2024-09-13T14:33:32.598196+0000","mapping_epoch":199,"log_start":"179'2700","ondisk_log_start":"179'2700","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"202'12079","last_scrub_stamp":"2024-09-13T14:33:31.042988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:33:31.042988+0000","objects_scrubbed":1589,"log_size":9388,"log_dups_size":0,"ondisk_log_size":9388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:43.765213+0000","scrub_duration":157,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103546880,"num_objects":1580,"num_object_clones":0,"num_object_copies":4740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230849+0000","last_change":"2024-09-13T14:24:00.652248+0000","last_active":"2024-09-13T14:33:31.230849+0000","last_peered":"2024-09-13T14:33:31.230849+0000","last_clean":"2024-09-13T14:33:31.230849+0000","last_became_active":"2024-09-13T14:24:00.651925+0000","last_became_peered":"2024-09-13T14:24:00.651925+0000","last_unstale":"2024-09-13T14:33:31.230849+0000","last_undegraded":"2024-09-13T14:33:31.230849+0000","last_fullsized":"2024-09-13T14:33:31.230849+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:04.913294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235359+0000","last_change":"2024-09-13T14:22:54.191581+0000","last_active":"2024-09-13T14:33:31.235359+0000","last_peered":"2024-09-13T14:33:31.235359+0000","last_clean":"2024-09-13T14:33:31.235359+0000","last_became_active":"2024-09-13T14:22:54.191202+0000","last_became_peered":"2024-09-13T14:22:54.191202+0000","last_unstale":"2024-09-13T14:33:31.235359+0000","last_undegraded":"2024-09-13T14:33:31.235359+0000","last_fullsized":"2024-09-13T14:33:31.235359+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:03.514767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.18","version":"204'12204","reported_seq":12256,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.929158+0000","last_change":"2024-09-13T14:33:23.254321+0000","last_active":"2024-09-13T14:33:32.929158+0000","last_peered":"2024-09-13T14:33:32.929158+0000","last_clean":"2024-09-13T14:33:32.929158+0000","last_became_active":"2024-09-13T14:33:23.233227+0000","last_became_peered":"2024-09-13T14:33:23.233227+0000","last_unstale":"2024-09-13T14:33:32.929158+0000","last_undegraded":"2024-09-13T14:33:32.929158+0000","last_fullsized":"2024-09-13T14:33:32.929158+0000","mapping_epoch":199,"log_start":"182'4400","ondisk_log_start":"182'4400","created":199,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:30:26.479756+0000","last_clean_scrub_stamp":"2024-09-13T14:30:26.479756+0000","objects_scrubbed":0,"log_size":7804,"log_dups_size":0,"ondisk_log_size":7804,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101351424,"num_objects":1547,"num_object_clones":0,"num_object_copies":4641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6253,"num_write_kb":249568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":185,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230854+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:33:31.230854+0000","last_peered":"2024-09-13T14:33:31.230854+0000","last_clean":"2024-09-13T14:33:31.230854+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:33:31.230854+0000","last_undegraded":"2024-09-13T14:33:31.230854+0000","last_fullsized":"2024-09-13T14:33:31.230854+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":153,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.235621+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:33:31.235621+0000","last_peered":"2024-09-13T14:33:31.235621+0000","last_clean":"2024-09-13T14:33:31.235621+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:33:31.235621+0000","last_undegraded":"2024-09-13T14:33:31.235621+0000","last_fullsized":"2024-09-13T14:33:31.235621+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":300,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.244715+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:33:34.244715+0000","last_peered":"2024-09-13T14:33:34.244715+0000","last_clean":"2024-09-13T14:33:34.244715+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:33:34.244715+0000","last_undegraded":"2024-09-13T14:33:34.244715+0000","last_fullsized":"2024-09-13T14:33:34.244715+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":286,"reported_epoch":206,"state":"active+clean","last_fresh":"2024-09-13T14:33:34.247534+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:33:34.247534+0000","last_peered":"2024-09-13T14:33:34.247534+0000","last_clean":"2024-09-13T14:33:34.247534+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:33:34.247534+0000","last_undegraded":"2024-09-13T14:33:34.247534+0000","last_fullsized":"2024-09-13T14:33:34.247534+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":178,"reported_epoch":203,"state":"active+clean","last_fresh":"2024-09-13T14:33:31.230507+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:33:31.230507+0000","last_peered":"2024-09-13T14:33:31.230507+0000","last_clean":"2024-09-13T14:33:31.230507+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:33:31.230507+0000","last_undegraded":"2024-09-13T14:33:31.230507+0000","last_fullsized":"2024-09-13T14:33:31.230507+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":276,"reported_epoch":204,"state":"active+clean","last_fresh":"2024-09-13T14:33:32.241291+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:33:32.241291+0000","last_peered":"2024-09-13T14:33:32.241291+0000","last_clean":"2024-09-13T14:33:32.241291+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:33:32.241291+0000","last_undegraded":"2024-09-13T14:33:32.241291+0000","last_fullsized":"2024-09-13T14:33:32.241291+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3331063832,"num_objects":50831,"num_object_clones":0,"num_object_copies":152493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4650,"num_objects_unfound":0,"num_objects_dirty":50831,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130654,"num_write_kb":5232929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9964761088,"data_stored":9964748872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":211164,"ondisk_log_size":211164,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397769,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1538456,"kb_used_data":1236936,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92206696,"statfs":{"total":95995035648,"available":94419656704,"internally_reserved":0,"allocated":1266622464,"data_stored":1265569735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593997,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339536,"kb_used_data":1029700,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92405616,"statfs":{"total":95995035648,"available":94623350784,"internally_reserved":0,"allocated":1054412800,"data_stored":1053372188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790227,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1319500,"kb_used_data":1013828,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92425652,"statfs":{"total":95995035648,"available":94643867648,"internally_reserved":0,"allocated":1038159872,"data_stored":1037115591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986455,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1342236,"kb_used_data":1028240,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92402916,"statfs":{"total":95995035648,"available":94620585984,"internally_reserved":0,"allocated":1052917760,"data_stored":1051865484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697013,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390060,"kb_used_data":2016408,"kb_used_omap":4,"kb_used_meta":373563,"kb_avail":91355092,"statfs":{"total":95995035648,"available":93547614208,"internally_reserved":0,"allocated":2064801792,"data_stored":2063754269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":382528587},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860540,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946680,"kb_used_data":1621548,"kb_used_omap":8,"kb_used_meta":325111,"kb_avail":91798472,"statfs":{"total":95995035648,"available":94001635328,"internally_reserved":0,"allocated":1660465152,"data_stored":1659406145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":332914508},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":91,"seq":390842024066,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335696,"kb_used_data":1034112,"kb_used_omap":7,"kb_used_meta":301560,"kb_avail":92409456,"statfs":{"total":95995035648,"available":94627282944,"internally_reserved":0,"allocated":1058930688,"data_stored":1057889953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7753,"internal_metadata":308797879},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187601,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1104068,"kb_used_data":817716,"kb_used_omap":7,"kb_used_meta":286328,"kb_avail":92641084,"statfs":{"total":95995035648,"available":94864470016,"internally_reserved":0,"allocated":837341184,"data_stored":836304296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":293200203},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":824119296,"data_stored":824115224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1052381184,"data_stored":1052377112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1657667584,"data_stored":1657667584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2056454144,"data_stored":2056454144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1045233664,"data_stored":1045233664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1038094336,"data_stored":1038090264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1044905984,"data_stored":1044905984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1245904896,"data_stored":1245904896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:33:38.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_1 pgp_num 26 2024-09-13T14:33:38.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:38.870 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:38.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:38.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:38.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:38.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:39.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:39.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:39.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:39.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:39.282 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:33:39.295 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:240552 5.18 5.e7d42f18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-13T14:33:39.298 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:18.527915+0000", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 260.76591488000003, 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94814983600000002, 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 240552 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527915+0000", 2024-09-13T14:33:39.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527915+0000", 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527914+0000", 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527919+0000", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8439999999999997e-06 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527927+0000", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2169999999999994e-06 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527931+0000", 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9179999999999996e-06 2024-09-13T14:33:39.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527971+0000", 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9838e-05 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.527979+0000", 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9519999999999994e-06 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476059+0000", 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94807948900000005 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476065+0000", 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5320000000000003e-06 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:240568 5.18 5.420dc278 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:18.545581+0000", 2024-09-13T14:33:39.303 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 260.74824894400001, 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93050964899999999, 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 240568 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545581+0000", 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545581+0000", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545580+0000", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545586+0000", 2024-09-13T14:33:39.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4790000000000004e-06 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545597+0000", 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0929e-05 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545603+0000", 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5590000000000002e-06 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545647+0000", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3614e-05 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.545653+0000", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3380000000000001e-06 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476085+0000", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93043221099999995 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476091+0000", 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5890000000000002e-06 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:240578 5.18 5.367f4478 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:18.551412+0000", 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 260.74241805600002, 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92469748900000004, 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:33:39.308 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 240578 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551412+0000", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551412+0000", 2024-09-13T14:33:39.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551412+0000", 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551414+0000", 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6089999999999999e-06 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551420+0000", 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5210000000000002e-06 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551422+0000", 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.492e-06 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551471+0000", 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8612000000000001e-05 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:18.551478+0000", 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8800000000000002e-06 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476102+0000", 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92462475799999999 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:19.476110+0000", 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0890000000000002e-06 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241012 5.19 5:98045e96:::benchmark_data_smithi017_147622_object106383:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486641+0000", 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.807188908, 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.249961208, 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241012 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486641+0000", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486641+0000", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486640+0000", 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486647+0000", 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6340000000000003e-06 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486661+0000", 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4076e-05 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486667+0000", 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3910000000000003e-06 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486720+0000", 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2689000000000003e-05 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op must wait for map", 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486727+0000", 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.490529+0000", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038017810000000002 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.490537+0000", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7710000000000006e-06 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.725390+0000", 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.234853583 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.725493+0000", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010329599999999999 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.725937+0000", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00044382999999999998 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.732692+0000", 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0067549840000000003 2024-09-13T14:33:39.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.732746+0000", 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3739000000000002e-05 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736431+0000", 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036852289999999999 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736441+0000", 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5699999999999999e-06 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736502+0000", 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1390999999999995e-05 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736536+0000", 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3918999999999998e-05 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736602+0000", 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6324999999999999e-05 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241017 5.19 5:9ab63f7f:::benchmark_data_smithi017_147622_object106388:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486677+0000", 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.807153148, 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2500717889999999, 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.322 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241017 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486677+0000", 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486677+0000", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486677+0000", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486680+0000", 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2320000000000001e-06 2024-09-13T14:33:39.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486687+0000", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3860000000000001e-06 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486692+0000", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3839999999999999e-06 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486780+0000", 2024-09-13T14:33:39.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8714000000000005e-05 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486785+0000", 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1780000000000002e-06 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491419+0000", 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046332750000000001 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491426+0000", 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1779999999999997e-06 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.732772+0000", 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2413457729999999 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.732870+0000", 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8026e-05 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.733252+0000", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00038204100000000001 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736381+0000", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031288140000000002 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736447+0000", 2024-09-13T14:33:39.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6068000000000005e-05 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736470+0000", 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3819999999999999e-05 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736637+0000", 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016606600000000001 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.329 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736667+0000", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0278e-05 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736692+0000", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5262999999999999e-05 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736749+0000", 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6740999999999998e-05 2024-09-13T14:33:39.330 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241019 5.19 5:9bc7a0be:::benchmark_data_smithi017_147622_object106390:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.80712320399999, 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.256814476, 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.331 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241019 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:33:39.332 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486706+0000", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486709+0000", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9229999999999998e-06 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486716+0000", 2024-09-13T14:33:39.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0690000000000004e-06 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486720+0000", 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9430000000000002e-06 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486807+0000", 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6104000000000001e-05 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486811+0000", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9119999999999997e-06 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491474+0000", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046621689999999999 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491479+0000", 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5219999999999998e-06 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736782+0000", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.245302446 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736872+0000", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0799999999999998e-05 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.737235+0000", 2024-09-13T14:33:39.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036251399999999998 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743274+0000", 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0060387319999999998 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743285+0000", 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1610000000000001e-05 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743340+0000", 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5067e-05 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743376+0000", 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6161999999999997e-05 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743417+0000", 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0441000000000001e-05 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743451+0000", 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3683999999999998e-05 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743521+0000", 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0822999999999996e-05 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241060 5.19 5:9f1c8886:::benchmark_data_smithi017_147622_object106431:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.339 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.80710526399997, 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.265054519, 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241060 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.340 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486724+0000", 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486727+0000", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.39e-06 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486745+0000", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8088e-05 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486751+0000", 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5740000000000003e-06 2024-09-13T14:33:39.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486922+0000", 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017172699999999999 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486928+0000", 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491499+0000", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045716660000000003 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491504+0000", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7899999999999999e-06 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.740366+0000", 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2488624189999999 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.740483+0000", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116347 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.741017+0000", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053390399999999998 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743385+0000", 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023684090000000001 2024-09-13T14:33:39.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743558+0000", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000173222 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751434+0000", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078755990000000005 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751517+0000", 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3270999999999996e-05 2024-09-13T14:33:39.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751580+0000", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2535999999999999e-05 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751671+0000", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1272000000000003e-05 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751779+0000", 2024-09-13T14:33:39.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010842500000000001 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241063 5.19 5:9e8b5f5a:::benchmark_data_smithi017_147622_object106434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.80698940000002, 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.272999937, 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.348 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241063 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486840+0000", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486849+0000", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2599999999999994e-06 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486860+0000", 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0482999999999999e-05 2024-09-13T14:33:39.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486865+0000", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.361e-06 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486954+0000", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9098999999999995e-05 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486959+0000", 2024-09-13T14:33:39.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0239999999999997e-06 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491524+0000", 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045650200000000004 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491529+0000", 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8980000000000004e-06 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743595+0000", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2520661099999999 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743721+0000", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000125845 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.744209+0000", 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00048819400000000001 2024-09-13T14:33:39.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751400+0000", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071904550000000001 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751834+0000", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00043387900000000002 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751878+0000", 2024-09-13T14:33:39.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4051999999999998e-05 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751911+0000", 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3725e-05 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759666+0000", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077542669999999996 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759719+0000", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3158000000000002e-05 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759841+0000", 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012194 2024-09-13T14:33:39.356 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241075 5.19 5:994d542c:::benchmark_data_smithi017_147622_object106446:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.80695974100001, 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2733477289999999, 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.357 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241075 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486873+0000", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3620000000000002e-06 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486881+0000", 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8469999999999996e-06 2024-09-13T14:33:39.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486885+0000", 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2039999999999999e-06 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486974+0000", 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9004000000000004e-05 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486977+0000", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2850000000000001e-06 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491625+0000", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046485279999999999 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491631+0000", 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7810000000000002e-06 2024-09-13T14:33:39.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747092+0000", 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2554612540000001 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747215+0000", 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012232599999999999 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747694+0000", 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00047920500000000002 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751946+0000", 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042521900000000003 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759556+0000", 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076104850000000002 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759602+0000", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5800999999999997e-05 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759899+0000", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00029672100000000001 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760044+0000", 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000145549 2024-09-13T14:33:39.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760096+0000", 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1566000000000001e-05 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760218+0000", 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012210199999999999 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.365 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241080 5.19 5:9eca9ef5:::benchmark_data_smithi017_147622_object106451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.525280+0000", 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.76855061399999, 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2352320940000001, 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241080 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.366 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525280+0000", 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525280+0000", 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525278+0000", 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525287+0000", 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3279999999999995e-06 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525298+0000", 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0772e-05 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525304+0000", 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8139999999999997e-06 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525348+0000", 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4715000000000001e-05 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:39.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.525357+0000", 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2230000000000007e-06 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751967+0000", 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.226609893 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.752080+0000", 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000112989 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.752611+0000", 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000531268 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759610+0000", 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069995459999999997 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759939+0000", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032867200000000001 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759997+0000", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8399000000000001e-05 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760247+0000", 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024943099999999999 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760344+0000", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7084999999999996e-05 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760389+0000", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5201999999999999e-05 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760512+0000", 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000122514 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.375 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241082 5.19 5:99a2c057:::benchmark_data_smithi017_147622_object106453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.528066+0000", 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.76576393599998, 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.232663032, 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.376 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241082 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528066+0000", 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528066+0000", 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528065+0000", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528071+0000", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7899999999999996e-06 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528084+0000", 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2928000000000001e-05 2024-09-13T14:33:39.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528089+0000", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.631e-06 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528131+0000", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1491e-05 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.528138+0000", 2024-09-13T14:33:39.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3599999999999998e-06 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.756196+0000", 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2280582339999999 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.756290+0000", 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3639000000000003e-05 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.756737+0000", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00044654500000000001 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759949+0000", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032119370000000002 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760283+0000", 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033394799999999998 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760309+0000", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6321000000000001e-05 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760565+0000", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000256001 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760610+0000", 2024-09-13T14:33:39.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5222999999999997e-05 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760664+0000", 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3594000000000002e-05 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760729+0000", 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5530000000000004e-05 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241117 5.19 5:9a0272af:::benchmark_data_smithi017_147622_object106488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.578090+0000", 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.715740213, 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2074867600000001, 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241117 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578090+0000", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578090+0000", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578089+0000", 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578095+0000", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3749999999999999e-06 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578107+0000", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2036e-05 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578113+0000", 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1340000000000001e-06 2024-09-13T14:33:39.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578158+0000", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4817000000000001e-05 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.578165+0000", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1779999999999997e-06 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760760+0000", 2024-09-13T14:33:39.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1825950889999999 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760857+0000", 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6675000000000005e-05 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.761312+0000", 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00045554500000000002 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768281+0000", 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069684660000000004 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768287+0000", 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3990000000000001e-06 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768315+0000", 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7478999999999999e-05 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.771869+0000", 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035545569999999999 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785403+0000", 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013533664000000001 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785454+0000", 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0682e-05 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785577+0000", 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123003 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241132 5.19 5:9ea549bd:::benchmark_data_smithi017_147622_object106503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.391 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.602382+0000", 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.69144796900002, 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.183446752, 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241132 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602382+0000", 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602382+0000", 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602381+0000", 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602388+0000", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0140000000000001e-06 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602399+0000", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0993e-05 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602403+0000", 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4009999999999999e-06 2024-09-13T14:33:39.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602443+0000", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9833999999999998e-05 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.602450+0000", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0500000000000003e-06 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.764735+0000", 2024-09-13T14:33:39.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.162284267 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.764865+0000", 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013032699999999999 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.765375+0000", 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00051033600000000001 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.771835+0000", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064595779999999997 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.771842+0000", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7610000000000002e-06 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.771894+0000", 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2843000000000002e-05 2024-09-13T14:33:39.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776652+0000", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047578309999999997 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785688+0000", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0090355149999999992 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785724+0000", 2024-09-13T14:33:39.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6668000000000002e-05 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785829+0000", 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010454999999999999 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241186 5.19 5:986d1346:::benchmark_data_smithi017_147622_object106557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.665380+0000", 2024-09-13T14:33:39.399 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.62844971499999, 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.120693816, 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241186 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665380+0000", 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665380+0000", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665379+0000", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665389+0000", 2024-09-13T14:33:39.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9920000000000002e-06 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665401+0000", 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2266000000000001e-05 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665407+0000", 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8200000000000002e-06 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665440+0000", 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3482000000000002e-05 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.665447+0000", 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7920000000000004e-06 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768334+0000", 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1028872940000001 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768445+0000", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000110522 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.768944+0000", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00049885000000000001 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.771938+0000", 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029945750000000002 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.772816+0000", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00087744099999999996 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776707+0000", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038914679999999999 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776744+0000", 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6293e-05 2024-09-13T14:33:39.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785936+0000", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091928440000000004 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785974+0000", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7146e-05 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.786074+0000", 2024-09-13T14:33:39.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010066299999999999 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241232 5.19 5:9deb177d:::benchmark_data_smithi017_147622_object106603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.721335+0000", 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.57249555700002, 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0689278449999999, 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.407 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241232 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721335+0000", 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721335+0000", 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721334+0000", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721337+0000", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4520000000000002e-06 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721344+0000", 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4930000000000003e-06 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721347+0000", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7989999999999998e-06 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721374+0000", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6945999999999999e-05 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.721380+0000", 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4740000000000002e-06 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.772842+0000", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0514622650000001 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.772961+0000", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118193 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.773444+0000", 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00048322000000000003 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776681+0000", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032374460000000002 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785637+0000", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0089554449999999994 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785662+0000", 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5338e-05 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.786486+0000", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00082398599999999997 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790079+0000", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003593073 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790139+0000", 2024-09-13T14:33:39.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0232999999999997e-05 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790262+0000", 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123139 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241234 5.19 5:9b3a63af:::benchmark_data_smithi017_147622_object106605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.725948+0000", 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.56788166699999, 2024-09-13T14:33:39.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0645499650000001, 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241234 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725948+0000", 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725948+0000", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725947+0000", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725953+0000", 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2600000000000002e-06 2024-09-13T14:33:39.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725965+0000", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1627999999999999e-05 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.725969+0000", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2640000000000002e-06 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.726004+0000", 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4798000000000001e-05 2024-09-13T14:33:39.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.726013+0000", 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4079999999999998e-06 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776770+0000", 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0507571069999999 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.776894+0000", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000124628 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.777432+0000", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053793600000000003 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785610+0000", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081780150000000003 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785884+0000", 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00027353499999999999 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785908+0000", 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4573999999999999e-05 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.789977+0000", 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040691410000000001 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790324+0000", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000346134 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790375+0000", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1561000000000003e-05 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790498+0000", 2024-09-13T14:33:39.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012330800000000001 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241242 5.19 5:9a0b9f62:::benchmark_data_smithi017_147622_object106613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.737075+0000", 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.55675511999999, 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0536455689999999, 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.422 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.423 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241242 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737075+0000", 2024-09-13T14:33:39.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737075+0000", 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737074+0000", 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737080+0000", 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2090000000000002e-06 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737090+0000", 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9459999999999993e-06 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737094+0000", 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1359999999999999e-06 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737132+0000", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8077999999999997e-05 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.737138+0000", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3579999999999999e-06 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.780723+0000", 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0435846289999999 2024-09-13T14:33:39.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.780861+0000", 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013842800000000001 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.781343+0000", 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00048156199999999999 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.785859+0000", 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045160690000000002 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.786109+0000", 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024968000000000001 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790024+0000", 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039154480000000002 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790041+0000", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.721e-05 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790545+0000", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00050328999999999999 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790591+0000", 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5837000000000002e-05 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790721+0000", 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000130068 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241264 5.19 5:9da90c75:::benchmark_data_smithi017_147622_object106635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.788847+0000", 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.50498275500001, 2024-09-13T14:33:39.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.014112288, 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241264 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788847+0000", 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788847+0000", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788846+0000", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788853+0000", 2024-09-13T14:33:39.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4429999999999997e-06 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788864+0000", 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0434999999999999e-05 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788870+0000", 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2539999999999998e-06 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788908+0000", 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7904000000000002e-05 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.788914+0000", 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8429999999999998e-06 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.786513+0000", 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99759917600000003 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.786673+0000", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015995399999999999 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.787219+0000", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00054638799999999997 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790764+0000", 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035444600000000001 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790799+0000", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5836e-05 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790833+0000", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.392e-05 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.794726+0000", 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038930589999999999 2024-09-13T14:33:39.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.802786+0000", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080590609999999993 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.802838+0000", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2663e-05 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.802960+0000", 2024-09-13T14:33:39.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012150999999999999 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241272 5.19 5:9c5eaf79:::benchmark_data_smithi017_147622_object106643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.825382+0000", 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 257.46844824200002, 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97787949900000004, 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.439 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241272 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825382+0000", 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825382+0000", 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825381+0000", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825387+0000", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.781e-06 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825399+0000", 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1131999999999999e-05 2024-09-13T14:33:39.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825403+0000", 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9310000000000001e-06 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825454+0000", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1919000000000002e-05 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.825465+0000", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0548e-05 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790855+0000", 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96538981000000001 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.790976+0000", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012086099999999999 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.791451+0000", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000474835 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.794827+0000", 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0033761730000000001 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.802750+0000", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0079231460000000007 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.802756+0000", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6329999999999999e-06 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803014+0000", 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025720499999999999 2024-09-13T14:33:39.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803046+0000", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2397999999999997e-05 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803100+0000", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3857999999999998e-05 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803261+0000", 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016148000000000001 2024-09-13T14:33:39.446 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241322 5.18 5:1f1fbe39:::benchmark_data_smithi017_147622_object106693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:22.803083+0000", 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 256.49074722400002, 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.920929675, 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241322 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.447 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803083+0000", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803083+0000", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803082+0000", 2024-09-13T14:33:39.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803087+0000", 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.276e-06 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803096+0000", 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5129999999999996e-06 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803101+0000", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4009999999999996e-06 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803136+0000", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4820999999999997e-05 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.803142+0000", 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.993e-06 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.595721+0000", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79257892299999999 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.595795+0000", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3850999999999995e-05 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,6", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.596004+0000", 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000208901 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.602880+0000", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068759939999999999 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.602916+0000", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6828e-05 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.723951+0000", 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12103504900000001 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.723989+0000", 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7738999999999999e-05 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:23.724013+0000", 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3433000000000001e-05 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:39.453 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:33:39.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:39.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:39.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:39.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:39.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[103001]: pgmap v593: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/151266 objects misplaced (2.612%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[103001]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[103001]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1213702674' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[107556]: pgmap v593: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/151266 objects misplaced (2.612%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[107556]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[107556]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1213702674' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.735 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:39 smithi189 ceph-mon[90992]: pgmap v593: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/151266 objects misplaced (2.612%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:39 smithi189 ceph-mon[90992]: 6.13 starting backfill to osd.0 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:39 smithi189 ceph-mon[90992]: 6.13 starting backfill to osd.3 from (0'0,0'0] MAX to 206'12253 2024-09-13T14:33:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1213702674' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:33:39.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:39.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:39.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:39.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:39.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:40.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:40.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:40.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:40.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:40.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:40.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:40.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:40 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:33:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:40 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:33:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:40 smithi189 ceph-mon[90992]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:33:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:40.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:40.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:40.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:33:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:33:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[103001]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:33:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:33:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:33:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:33:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:40 smithi017 ceph-mon[107556]: osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:33:40.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:40.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:40.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:40.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:40.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:41.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:41.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:41.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:41.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:41.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:41.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:41.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:41.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:41.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:41.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:41 smithi017 ceph-mon[103001]: pgmap v596: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/149070 objects misplaced (2.650%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:41 smithi017 ceph-mon[107556]: pgmap v596: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/149070 objects misplaced (2.650%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:41 smithi189 ceph-mon[90992]: pgmap v596: 109 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 107 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 3951/149070 objects misplaced (2.650%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:33:41.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:41.789 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:41.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:33:41.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:41.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:41.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:42.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:42.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:42.103 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:42.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:42.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:42.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:42.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:42.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:42.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:42.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:42.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:42 smithi189 ceph-mon[90992]: 6.18 scrub starts 2024-09-13T14:33:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:42 smithi189 ceph-mon[90992]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:33:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:42 smithi189 ceph-mon[90992]: 6.18 scrub ok 2024-09-13T14:33:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.521 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:42.543 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:42.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:42.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[103001]: 6.18 scrub starts 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[103001]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[103001]: 6.18 scrub ok 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[107556]: 6.18 scrub starts 2024-09-13T14:33:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[107556]: osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:33:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[107556]: 6.18 scrub ok 2024-09-13T14:33:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:42.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:42.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:42.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:42.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:43.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:43.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:43.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:43.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:43.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:43.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:43.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:43.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:43.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:43.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:43.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:43.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:43 smithi189 ceph-mon[90992]: pgmap v598: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 activating+remapped, 106 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6961/147912 objects misplaced (4.706%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:33:43.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:43 smithi189 ceph-mon[90992]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:33:43.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:43.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:43 smithi017 ceph-mon[107556]: pgmap v598: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 activating+remapped, 106 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6961/147912 objects misplaced (4.706%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:33:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:43 smithi017 ceph-mon[107556]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:33:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:43 smithi017 ceph-mon[103001]: pgmap v598: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 activating+remapped, 106 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6961/147912 objects misplaced (4.706%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:33:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:43 smithi017 ceph-mon[103001]: osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:33:43.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:43.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:43.902 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:33:43.902 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:33:43.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:203794 6.a 6:59764ec8:::benchmark_data_smithi017_185083_object79182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:43.892151+0000", 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0080911479999999994, 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081185019999999997, 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 203794 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:43.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892151+0000", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892151+0000", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892149+0000", 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:33:43.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892155+0000", 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2079999999999997e-06 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892171+0000", 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.523e-05 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.892176+0000", 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8069999999999996e-06 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895146+0000", 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029694779999999998 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895315+0000", 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016928399999999999 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895448+0000", 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013274000000000001 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:203798 6.17 6:e9801328:::benchmark_data_smithi017_185083_object79186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:43.895825+0000", 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0044168050000000002, 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045321909999999997, 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 203798 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895825+0000", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895825+0000", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895816+0000", 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999914 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895829+0000", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2241e-05 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895844+0000", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5729000000000001e-05 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895850+0000", 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.643e-06 2024-09-13T14:33:43.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.895900+0000", 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9589999999999998e-05 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.896007+0000", 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000107658 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:33:43.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.896365+0000", 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035762199999999999 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:203799 6.7 6:e50c31fe:::benchmark_data_smithi017_185083_object79187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:43.897322+0000", 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0029199959999999998, 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030838860000000001, 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:33:43.912 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 203799 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897322+0000", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897322+0000", 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:33:43.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897321+0000", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897325+0000", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8999999999999997e-06 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897336+0000", 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0661e-05 2024-09-13T14:33:43.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.897341+0000", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2989999999999996e-06 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.899941+0000", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025992620000000002 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:43.900051+0000", 2024-09-13T14:33:43.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000110398 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 3 2024-09-13T14:33:43.916 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:33:43.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:43.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:43.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:43.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:44.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:44.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:44.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:44.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:44.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:44.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:44 smithi017 ceph-mon[107556]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:33:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:33:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:44 smithi017 ceph-mon[103001]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:33:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:33:44.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:44 smithi189 ceph-mon[90992]: osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:33:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:33:44.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:44.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:44.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:44.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:44.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:33:44.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:44.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:44.860 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:44.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:44.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:44.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:44.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:44.946 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:45.005 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:45.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:45.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:45.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:45.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:45.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:45.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:45.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:33:45.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:45.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:45.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[107556]: pgmap v601: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5401/140373 objects misplaced (3.848%); 21 MiB/s, 332 objects/s recovering 2024-09-13T14:33:45.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:33:45.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[107556]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:33:45.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[107556]: 6.7 scrub starts 2024-09-13T14:33:45.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[107556]: 6.7 scrub ok 2024-09-13T14:33:45.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[103001]: pgmap v601: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5401/140373 objects misplaced (3.848%); 21 MiB/s, 332 objects/s recovering 2024-09-13T14:33:45.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:33:45.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[103001]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:33:45.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[103001]: 6.7 scrub starts 2024-09-13T14:33:45.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:45 smithi017 ceph-mon[103001]: 6.7 scrub ok 2024-09-13T14:33:45.447 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:33:45.448 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:33:45.471 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:33:45.471 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:33:45.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:33:45.494 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:33:45.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:45.520 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:33:45.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:33:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:45.544 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:33:45.544 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:33:45.567 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:33:45.567 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:33:45.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:33:45.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:33:45.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:33:45.615 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:33:45.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:45.657 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:45 smithi189 ceph-mon[90992]: pgmap v601: 109 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 107 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5401/140373 objects misplaced (3.848%); 21 MiB/s, 332 objects/s recovering 2024-09-13T14:33:45.658 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:45 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:33:45.658 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:45 smithi189 ceph-mon[90992]: osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:33:45.658 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:45 smithi189 ceph-mon[90992]: 6.7 scrub starts 2024-09-13T14:33:45.658 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:45 smithi189 ceph-mon[90992]: 6.7 scrub ok 2024-09-13T14:33:45.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:45.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:45.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:45.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:45.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:45.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:46.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:46.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:46.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:46.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:46.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:46.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:46.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:46.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:46 smithi189 ceph-mon[90992]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:33:46.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:46 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/251382199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:46.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:46 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:33:46.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:46.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:46.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:46.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[103001]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:33:46.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/251382199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:46.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:33:46.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[107556]: osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:33:46.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/251382199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:46.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:46 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:33:46.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:46.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:46.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:46.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:46.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:46.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:46.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:47.030 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:47.045 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:47.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:47.140 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:33:47.140 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:33:47.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:47.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:47.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:47.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:47.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: pgmap v604: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3010/137658 objects misplaced (2.187%); 36 MiB/s, 575 objects/s recovering 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: pgmap v604: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3010/137658 objects misplaced (2.187%); 36 MiB/s, 575 objects/s recovering 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:47.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:47] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T14:33:47.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:47.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:47.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:47.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:47.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: pgmap v604: 109 pgs: 1 active+remapped+backfill_wait, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 3010/137658 objects misplaced (2.187%); 36 MiB/s, 575 objects/s recovering 2024-09-13T14:33:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:33:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:33:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: 6.14 starting backfill to osd.2 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: 6.14 starting backfill to osd.4 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: 6.14 starting backfill to osd.7 from (0'0,0'0] MAX to 214'12527 2024-09-13T14:33:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:47 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:47.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:33:47.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:47.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:47.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:47.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:47.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:47.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:48.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:48.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:48.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:48.105 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:48.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:48.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:48.334 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:48.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:48.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:33:48.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:48 smithi189 ceph-mon[90992]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:33:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:48 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:33:48.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:48.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:48.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:48.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:48 smithi017 ceph-mon[103001]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:33:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:48 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:33:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:48 smithi017 ceph-mon[107556]: osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:33:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:48 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:33:48.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:48.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:48.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:48.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:48.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:48.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:49.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:49.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:49.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:49.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:49.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:49.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:49.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:49.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:49.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:49.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:49.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:49 smithi189 ceph-mon[90992]: pgmap v607: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 5962/135771 objects misplaced (4.391%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:33:49.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:49 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:33:49.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:49 smithi189 ceph-mon[90992]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:33:49.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:49 smithi189 ceph-mon[90992]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 215'12168 2024-09-13T14:33:49.626 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:49 smithi189 ceph-mon[90992]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:33:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[103001]: pgmap v607: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 5962/135771 objects misplaced (4.391%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:33:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:33:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[103001]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:33:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[103001]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 215'12168 2024-09-13T14:33:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[103001]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:33:49.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[107556]: pgmap v607: 109 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 107 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 5962/135771 objects misplaced (4.391%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:33:49.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:33:49.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[107556]: osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:33:49.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[107556]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 215'12168 2024-09-13T14:33:49.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:49 smithi017 ceph-mon[107556]: osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:33:49.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:49.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:49.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:49.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:49.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:49.862 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:33:49.862 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:33:49.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:49.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:49.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:49.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:50.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:50.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:50.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:50.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:50.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:50.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:50.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:33:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:33:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:50.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:50.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:50.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:33:50.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.691 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:50.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:50.759 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:50 smithi017 ceph-mon[103001]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:33:50.759 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:50 smithi017 ceph-mon[103001]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 218'12296 2024-09-13T14:33:50.761 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:50 smithi017 ceph-mon[107556]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:33:50.761 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:50 smithi017 ceph-mon[107556]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 218'12296 2024-09-13T14:33:50.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:50.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:50.903 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:50.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:50.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:50.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:50.964 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:50 smithi189 ceph-mon[90992]: osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:33:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:50 smithi189 ceph-mon[90992]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 218'12296 2024-09-13T14:33:51.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:51.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:51.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:51.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:51.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:51.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:51.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:51.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:51.519 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 6 pgp_num to 26 2024-09-13T14:33:51.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:51.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[103001]: pgmap v611: 109 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4349/131658 objects misplaced (3.303%); 32 MiB/s, 509 objects/s recovering 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[103001]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[103001]: 6.17 deep-scrub starts 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2267440785' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[107556]: pgmap v611: 109 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4349/131658 objects misplaced (3.303%); 32 MiB/s, 509 objects/s recovering 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[107556]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[107556]: 6.17 deep-scrub starts 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2267440785' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:51 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:51.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:51.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:51.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:51 smithi189 ceph-mon[90992]: pgmap v611: 109 pgs: 1 activating+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4349/131658 objects misplaced (3.303%); 32 MiB/s, 509 objects/s recovering 2024-09-13T14:33:51.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:51 smithi189 ceph-mon[90992]: osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:33:51.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:51 smithi189 ceph-mon[90992]: 6.17 deep-scrub starts 2024-09-13T14:33:51.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2267440785' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.782 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:51 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T14:33:51.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:51.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:51.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:51.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:52.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:52.141 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.174 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:52.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:52.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:52.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:52.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:52.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:52.531 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:33:52.540 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:52.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:52.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:52.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:52.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:52.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:52.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:52.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T14:33:52.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[103001]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:33:52.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:33:52.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T14:33:52.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[107556]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:33:52.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:33:52.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:52.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:52.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:52 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T14:33:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:52 smithi189 ceph-mon[90992]: osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:33:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:33:53.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:53.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:53.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.272 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:53.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:53.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:53.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:53.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:53.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:53.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:53.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:53.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:53.723 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:53 smithi189 ceph-mon[90992]: pgmap v614: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 1397/131148 objects misplaced (1.065%); 43 MiB/s, 685 objects/s recovering 2024-09-13T14:33:53.723 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:53 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:53.723 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:33:53.723 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:53 smithi189 ceph-mon[90992]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:33:53.764 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:33:53.764 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:33:53.784 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[103001]: pgmap v614: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 1397/131148 objects misplaced (1.065%); 43 MiB/s, 685 objects/s recovering 2024-09-13T14:33:53.784 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[103001]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[107556]: pgmap v614: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 1397/131148 objects misplaced (1.065%); 43 MiB/s, 685 objects/s recovering 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1896431900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:33:53.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:53 smithi017 ceph-mon[107556]: osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:33:53.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:53.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:53.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:53.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:53.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:54.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:54.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:54.201 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:54.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:54.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:54.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:54.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:33:54.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:54.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:54.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:54.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3549531659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3549531659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:54.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3549531659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:33:54.776 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:54.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:54.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:54.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:54.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:55.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:55.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:33:55.077 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:33:55.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:33:55.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:55.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:55.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:55.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:55.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:55.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:55.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:55.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:55.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:33:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:55.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:55.780 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:55.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:55.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:55 smithi017 ceph-mon[103001]: pgmap v616: 109 pgs: 1 active+remapped+backfilling, 1 peering, 1 activating+remapped, 106 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1551/124503 objects misplaced (1.246%); 50 MiB/s, 807 objects/s recovering 2024-09-13T14:33:55.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:55 smithi017 ceph-mon[103001]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:33:55.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:55 smithi017 ceph-mon[107556]: pgmap v616: 109 pgs: 1 active+remapped+backfilling, 1 peering, 1 activating+remapped, 106 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1551/124503 objects misplaced (1.246%); 50 MiB/s, 807 objects/s recovering 2024-09-13T14:33:55.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:55 smithi017 ceph-mon[107556]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:33:55.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:55.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:55.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:55.937 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:55 smithi189 ceph-mon[90992]: pgmap v616: 109 pgs: 1 active+remapped+backfilling, 1 peering, 1 activating+remapped, 106 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1551/124503 objects misplaced (1.246%); 50 MiB/s, 807 objects/s recovering 2024-09-13T14:33:55.937 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:55 smithi189 ceph-mon[90992]: osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:33:56.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:56.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:56.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:56.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:56.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:56.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:56.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:56.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:56.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:56.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:56.662 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:56.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:56 smithi189 ceph-mon[90992]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:33:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:56.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:56.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:56.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:56.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:56.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:56.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:56 smithi017 ceph-mon[103001]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:33:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:56 smithi017 ceph-mon[107556]: osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:33:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:57.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:57.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:57.102 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:33:57.103 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:33:57.103 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:33:57.103 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:33:57.103 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:33:57.149 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:57.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:57.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:57.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:57.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:33:57.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:33:57.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:57.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:57.557 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:33:57] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T14:33:57.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:57.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:57.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:57 smithi189 ceph-mon[90992]: pgmap v619: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 519/120864 objects misplaced (0.429%); 29 MiB/s, 469 objects/s recovering 2024-09-13T14:33:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:57 smithi189 ceph-mon[90992]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:33:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:57.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:57.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:33:57.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:57.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:57.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[103001]: pgmap v619: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 519/120864 objects misplaced (0.429%); 29 MiB/s, 469 objects/s recovering 2024-09-13T14:33:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[103001]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:33:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[107556]: pgmap v619: 109 pgs: 2 active+remapped+backfilling, 1 peering, 106 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 519/120864 objects misplaced (0.429%); 29 MiB/s, 469 objects/s recovering 2024-09-13T14:33:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[107556]: osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:33:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:33:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:58.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:58.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:58.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:58.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:58.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:58.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:58.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:58.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:58.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:58.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:58.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:33:58.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:33:58.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:58.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:58.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:58.779 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:33:58.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:58.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:58.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:58.916 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:33:58.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:33:58.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:33:58.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:33:59.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:59.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:33:59.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:33:59.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:59.130 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:59.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:33:59.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:59.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:33:59.360 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:33:59.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:33:59.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:59.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:59.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:33:59.627 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:33:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[103001]: pgmap v621: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 154/117018 objects misplaced (0.132%); 28 MiB/s, 450 objects/s recovering 2024-09-13T14:33:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:33:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[103001]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:33:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[107556]: pgmap v621: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 154/117018 objects misplaced (0.132%); 28 MiB/s, 450 objects/s recovering 2024-09-13T14:33:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:33:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:33:59 smithi017 ceph-mon[107556]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:33:59.714 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:33:59.714 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:33:59.715 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:33:59.715 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T14:33:59.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1 2024-09-13T14:33:59.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:33:59.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:33:59.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:33:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:59 smithi189 ceph-mon[90992]: pgmap v621: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 154/117018 objects misplaced (0.132%); 28 MiB/s, 450 objects/s recovering 2024-09-13T14:33:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:33:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:33:59 smithi189 ceph-mon[90992]: osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:33:59.813 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:59.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:33:59.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:33:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:33:59.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:33:59.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:00.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:00.048 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:33:59 smithi017 systemd[1]: Stopping Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:34:00.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:00.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:00.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:00.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:00.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:00.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:00.449 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:00.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:00.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:00.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:00.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:00.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[103001]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:34:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[103001]: pgmap v624: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:34:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:34:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[107556]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:34:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[107556]: pgmap v624: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:34:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:34:00.719 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:00 smithi189 ceph-mon[90992]: osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:34:00.719 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:00 smithi189 ceph-mon[90992]: pgmap v624: 109 pgs: 109 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:34:00.719 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:34:00.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:00.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:00.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:00.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:00.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:01.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.048 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:01.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.176 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179068 5.11 5.50a6af91 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.60858786799997, 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80406319500000001, 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.177 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179068 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562183+0000", 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562189+0000", 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4139999999999999e-06 2024-09-13T14:34:01.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562203+0000", 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3096e-05 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562209+0000", 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4710000000000004e-06 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562251+0000", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2101999999999998e-05 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562258+0000", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8499999999999996e-06 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366237+0000", 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80397893600000003 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366248+0000", 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1025e-05 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179072 5.11 5.a9629bd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.60740324300002, 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80290814399999999, 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.182 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179072 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:34:01.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563368+0000", 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563373+0000", 2024-09-13T14:34:01.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9019999999999998e-06 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563385+0000", 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1528e-05 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563390+0000", 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7400000000000001e-06 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563593+0000", 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000202559 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563599+0000", 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366269+0000", 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80267021500000002 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366277+0000", 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6750000000000008e-06 2024-09-13T14:34:01.188 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179268 5.11 5.957b59d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.38212263600002, 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.57765650000000002, 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.189 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179268 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.190 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788648+0000", 2024-09-13T14:34:01.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788655+0000", 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7329999999999997e-06 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788668+0000", 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.296e-05 2024-09-13T14:34:01.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788673+0000", 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7089999999999999e-06 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.798794+0000", 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010120891999999999 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.798801+0000", 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6980000000000001e-06 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366298+0000", 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.56749667100000001 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366306+0000", 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6989999999999992e-06 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179352 5.11 5.b7d57211 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.27525276699998, 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.47084923899999997, 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.195 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179352 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:34:01.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895518+0000", 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895523+0000", 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4399999999999998e-06 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.198 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895531+0000", 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5399999999999996e-06 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895536+0000", 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7040000000000002e-06 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.897591+0000", 2024-09-13T14:34:01.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002054967 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.897598+0000", 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6109999999999999e-06 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366360+0000", 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.46876190000000001 2024-09-13T14:34:01.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366369+0000", 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.499e-06 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179409 5.11 5.eddc87b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.201 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.211682677, 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40730723800000002, 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179409 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.202 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.203 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959088+0000", 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959096+0000", 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9650000000000008e-06 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959107+0000", 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0278999999999999e-05 2024-09-13T14:34:01.204 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959112+0000", 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3750000000000002e-06 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960225+0000", 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011133849999999999 2024-09-13T14:34:01.205 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960278+0000", 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2590000000000003e-05 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366391+0000", 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40611253200000003 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.206 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366397+0000", 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5729999999999996e-06 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179414 5.11 5.998ec5d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.20989354599999, 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40555330499999998, 2024-09-13T14:34:01.207 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179414 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960878+0000", 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960883+0000", 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5179999999999998e-06 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960895+0000", 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2782000000000001e-05 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960901+0000", 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0649999999999998e-06 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.961050+0000", 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014901500000000001 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.961055+0000", 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2059999999999998e-06 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366423+0000", 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40536831699999998 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366432+0000", 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3070000000000008e-06 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.213 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179424 5.11 5.a52315d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.19462419500002, 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.39031132600000001, 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.214 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179424 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.215 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976147+0000", 2024-09-13T14:34:01.216 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976153+0000", 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4649999999999999e-06 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976164+0000", 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0903e-05 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976170+0000", 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8599999999999998e-06 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976365+0000", 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000194673 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976371+0000", 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9120000000000003e-06 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366453+0000", 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.39008283999999999 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366460+0000", 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3090000000000001e-06 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179473 5.11 5.b26654d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.11331592200003, 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30902944900000001, 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.220 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179473 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.221 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057456+0000", 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057462+0000", 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8989999999999998e-06 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.222 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057472+0000", 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.077e-05 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057478+0000", 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0859999999999998e-06 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057676+0000", 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019763699999999999 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057682+0000", 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1310000000000003e-06 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366479+0000", 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30879694600000002 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366486+0000", 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9789999999999996e-06 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179488 5.11 5.79892fd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 358.09235632799999, 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28810034800000001, 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.225 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179488 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078423+0000", 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5650000000000004e-06 2024-09-13T14:34:01.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078436+0000", 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.324e-05 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078442+0000", 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4079999999999997e-06 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078806+0000", 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036393999999999999 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078812+0000", 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1800000000000001e-06 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366506+0000", 2024-09-13T14:34:01.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.287694279 2024-09-13T14:34:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:01.231 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366517+0000", 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0479000000000001e-05 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194100 5.11 5:899dff21:::benchmark_data_smithi017_147622_object59471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.68346038800001, 2024-09-13T14:34:01.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93940912399999998, 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194100 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487310+0000", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487320+0000", 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8600000000000005e-06 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487332+0000", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2334000000000001e-05 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487338+0000", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.049e-06 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487503+0000", 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016442900000000001 2024-09-13T14:34:01.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487511+0000", 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421662+0000", 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93415139400000002 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.236 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421760+0000", 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8265999999999997e-05 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421840+0000", 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9374000000000003e-05 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426549+0000", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047089769999999996 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426575+0000", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6684000000000001e-05 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426589+0000", 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4156e-05 2024-09-13T14:34:01.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426686+0000", 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6089999999999996e-05 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426721+0000", 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5760000000000003e-05 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194109 5.11 5:8fec395f:::benchmark_data_smithi017_147622_object59480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.66972263999997, 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93220517000000003, 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194109 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.240 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501048+0000", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501056+0000", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4189999999999996e-06 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501073+0000", 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7646e-05 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501081+0000", 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6510000000000001e-06 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501133+0000", 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1722999999999999e-05 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501155+0000", 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1834000000000001e-05 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423537+0000", 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92238267699999998 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.244 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423686+0000", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000148648 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423774+0000", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8084999999999994e-05 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.428865+0000", 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0050907490000000003 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433082+0000", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042173330000000002 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433133+0000", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1220000000000001e-05 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433218+0000", 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4188000000000001e-05 2024-09-13T14:34:01.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433255+0000", 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7561999999999997e-05 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194177 5.11 5:8c2311cb:::benchmark_data_smithi017_147622_object59548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.60462276099997, 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86727486499999995, 2024-09-13T14:34:01.247 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194177 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566148+0000", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566154+0000", 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0589999999999999e-06 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566168+0000", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3652000000000001e-05 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566174+0000", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5260000000000001e-06 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566217+0000", 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3430000000000003e-05 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566224+0000", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2420000000000003e-06 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425124+0000", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.858899682 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425232+0000", 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010812899999999999 2024-09-13T14:34:01.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425339+0000", 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010717699999999999 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433108+0000", 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077691629999999999 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433327+0000", 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000218097 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433334+0000", 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8229999999999995e-06 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433380+0000", 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5207000000000003e-05 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433425+0000", 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5126999999999997e-05 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194205 5.11 5:8ab11436:::benchmark_data_smithi017_147622_object59576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.57461963700001, 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84416819300000001, 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194205 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596151+0000", 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596158+0000", 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1779999999999997e-06 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596172+0000", 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3178e-05 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596177+0000", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5999999999999997e-06 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596216+0000", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9270999999999997e-05 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596225+0000", 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2139999999999996e-06 2024-09-13T14:34:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426749+0000", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83052429999999999 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426843+0000", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3908999999999998e-05 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426946+0000", 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103627 2024-09-13T14:34:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433290+0000", 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006343591 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433477+0000", 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018701699999999999 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440230+0000", 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006752699 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440282+0000", 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2042000000000001e-05 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440321+0000", 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9345999999999999e-05 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194221 5.11 5:8ecddf2c:::benchmark_data_smithi017_147622_object59592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.55909304800002, 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83324189699999995, 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194221 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:34:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611678+0000", 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611689+0000", 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0625000000000001e-05 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611700+0000", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0900999999999999e-05 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611706+0000", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0809999999999999e-06 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611749+0000", 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3550000000000001e-05 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611758+0000", 2024-09-13T14:34:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.918e-06 2024-09-13T14:34:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.428926+0000", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.81716769499999997 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.429052+0000", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012662799999999999 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.429167+0000", 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000114589 2024-09-13T14:34:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440185+0000", 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011018035000000001 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440329+0000", 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000143702 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444827+0000", 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0044977840000000003 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444882+0000", 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5121999999999998e-05 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444921+0000", 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9728999999999999e-05 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194323 5.11 5:8858a9dc:::benchmark_data_smithi017_147622_object59694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.43653591899999, 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71087829400000002, 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194323 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734235+0000", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734242+0000", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4599999999999997e-06 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734260+0000", 2024-09-13T14:34:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7839e-05 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734267+0000", 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.092e-06 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734313+0000", 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6047000000000001e-05 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734322+0000", 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9949999999999994e-06 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431146+0000", 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69682316600000005 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431247+0000", 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101891 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431358+0000", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000110159 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440214+0000", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0088563949999999995 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440332+0000", 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118144 2024-09-13T14:34:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445019+0000", 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046870799999999997 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445076+0000", 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7064000000000001e-05 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445115+0000", 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8634999999999999e-05 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194411 5.11 5:8901ba5a:::benchmark_data_smithi017_147622_object59782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.33093024599998, 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.60801552800000003, 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194411 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839841+0000", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839849+0000", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4710000000000007e-06 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839867+0000", 2024-09-13T14:34:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7567e-05 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839874+0000", 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1130000000000003e-06 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839941+0000", 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7269999999999998e-05 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839950+0000", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8890000000000008e-06 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433500+0000", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.59354982199999995 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433593+0000", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3528000000000003e-05 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433690+0000", 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6317999999999995e-05 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440359+0000", 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066693330000000004 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442529+0000", 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021701609999999999 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447702+0000", 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0051729669999999997 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447810+0000", 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010777699999999999 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447858+0000", 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8075000000000003e-05 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194483 5.11 5:8deb5587:::benchmark_data_smithi017_147622_object59854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.234407659, 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51168034699999998, 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194483 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:34:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936363+0000", 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936371+0000", 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2179999999999999e-06 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936386+0000", 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4722000000000001e-05 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936404+0000", 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7702000000000001e-05 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936451+0000", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7008000000000001e-05 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936458+0000", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9209999999999992e-06 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435539+0000", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.49908090999999999 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435637+0000", 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7547999999999995e-05 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435734+0000", 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7188000000000005e-05 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442568+0000", 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068333930000000001 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444770+0000", 2024-09-13T14:34:01.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022027959999999999 2024-09-13T14:34:01.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447888+0000", 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031180650000000002 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448006+0000", 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011807399999999999 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448045+0000", 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8838000000000003e-05 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194542 5.11 5:891d791e:::benchmark_data_smithi017_147622_object59913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.14864622200002, 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.42990321399999998, 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.294 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194542 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022125+0000", 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022132+0000", 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1380000000000001e-06 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022150+0000", 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8580000000000002e-05 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022158+0000", 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6930000000000007e-06 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022213+0000", 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4965e-05 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022221+0000", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6790000000000005e-06 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437665+0000", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.41544472399999999 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437763+0000", 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7056e-05 2024-09-13T14:34:01.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437865+0000", 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010253 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442572+0000", 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047068960000000003 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444968+0000", 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023965150000000001 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.451923+0000", 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069545290000000001 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.451980+0000", 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7089999999999997e-05 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452030+0000", 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9486e-05 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194600 5.11 5:8be3cac2:::benchmark_data_smithi017_147622_object59971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 337.078829472, 2024-09-13T14:34:01.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.36026992099999999, 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194600 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091941+0000", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091951+0000", 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0305999999999999e-05 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091974+0000", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2739e-05 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091982+0000", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2889999999999998e-06 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.092045+0000", 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3238000000000007e-05 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.092053+0000", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8699999999999992e-06 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440389+0000", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34833567799999998 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440474+0000", 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5704000000000005e-05 2024-09-13T14:34:01.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440580+0000", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000105441 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444934+0000", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004354485 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445181+0000", 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024614100000000001 2024-09-13T14:34:01.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452150+0000", 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069696760000000002 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452177+0000", 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7010999999999999e-05 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452213+0000", 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5794000000000003e-05 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194672 5.11 5:896b780c:::benchmark_data_smithi017_147622_object60043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 336.96703086299999, 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24859889499999999, 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:34:01.308 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194672 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203741+0000", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203747+0000", 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9450000000000004e-06 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203761+0000", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3733999999999999e-05 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203768+0000", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5799999999999997e-06 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203816+0000", 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8355e-05 2024-09-13T14:34:01.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203834+0000", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8175000000000001e-05 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443084+0000", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.23924999799999999 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443204+0000", 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011947699999999999 2024-09-13T14:34:01.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443324+0000", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011992199999999999 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447175+0000", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038516660000000001 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448258+0000", 2024-09-13T14:34:01.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001082246 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452258+0000", 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004000417 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452306+0000", 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7936999999999999e-05 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:34:01.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452341+0000", 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4644000000000002e-05 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:34:01.315 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:34:01.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:01.354 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:01.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:01.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:01.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:01.581 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:34:01.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:01.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:01.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:01.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:01.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:34:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[107556]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:34:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[107556]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 227'12784 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1734487803' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[103001]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[103001]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 227'12784 2024-09-13T14:34:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1734487803' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:01.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:01 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:01.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:01.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:01.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:34:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:01 smithi189 ceph-mon[90992]: osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:34:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:01 smithi189 ceph-mon[90992]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 227'12784 2024-09-13T14:34:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1734487803' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:01 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:02.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:02.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:34:02.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:02.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:02.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:02.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:02.481 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:02.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:02.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:02.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:02.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:02 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:02 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:02 smithi189 ceph-mon[90992]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:34:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:02 smithi189 ceph-mon[90992]: pgmap v627: 109 pgs: 109 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:34:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:34:02.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:02.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[107556]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[107556]: pgmap v627: 109 pgs: 109 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[103001]: osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[103001]: pgmap v627: 109 pgs: 109 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:34:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:34:02.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:02.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:02.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:02.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:03.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:03.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:03.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:03.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:03.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:03.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:03.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:03.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:03.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:03.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:03.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:03.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:03.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:03.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:03.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:03.883 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:34:03.883 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:03 smithi189 ceph-mon[90992]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:34:03.883 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:03 smithi189 ceph-mon[90992]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.883 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:03 smithi189 ceph-mon[90992]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.883 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:03 smithi189 ceph-mon[90992]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:34:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:34:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[107556]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:34:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[107556]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[107556]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[107556]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:34:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:34:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[103001]: osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:34:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[103001]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[103001]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 228'12659 2024-09-13T14:34:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:03 smithi017 ceph-mon[103001]: osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:34:03.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:04.008 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:04.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:04.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:04.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:04.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:04.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:04.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:04.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:04.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:04.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:04.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:04.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:04.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:04.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:04.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:04.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:04.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:05.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:05.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:05.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:05.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:05.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:05.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:34:05.299 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:34:05.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:05.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:05.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:34:05.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:34:05.391 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:34:05.391 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:34:05.440 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:34:05.441 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:34:05.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:05.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:34:05.468 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:34:05.503 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:34:05.503 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:34:05.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:05.540 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:34:05.540 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:34:05.571 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:34:05.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:34:05.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:05.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:05.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: pgmap v630: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 89/104058 objects misplaced (0.086%); 16 MiB/s, 257 objects/s recovering 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1519434676' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: pgmap v630: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 89/104058 objects misplaced (0.086%); 16 MiB/s, 257 objects/s recovering 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1519434676' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:05.703 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:34:05.561+0000 7fb0e8ff2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:34:05.703 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:34:05.561+0000 7fb0e8ff2640 -1 osd.1 234 *** Got signal Terminated *** 2024-09-13T14:34:05.703 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[121839]: 2024-09-13T14:34:05.561+0000 7fb0e8ff2640 -1 osd.1 234 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:34:05.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: pgmap v630: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 89/104058 objects misplaced (0.086%); 16 MiB/s, 257 objects/s recovering 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 230'12620 2024-09-13T14:34:05.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1519434676' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:05.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:05.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:05.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:05.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:06.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:06.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:06.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:06.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:06.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:06.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:06.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:06.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:06.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[103001]: osd.1 marked itself down and dead 2024-09-13T14:34:06.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.639 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[107556]: osd.1 marked itself down and dead 2024-09-13T14:34:06.639 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.639 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:06 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:06.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:06 smithi189 ceph-mon[90992]: osd.1 marked itself down and dead 2024-09-13T14:34:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:06 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:06 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:06.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:06.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:06.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:06.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:06.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:07.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:07.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:07.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:07.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:07.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:07.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:07.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:34:07.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:07.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:07.530 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:07] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T14:34:07.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[107556]: pgmap v632: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 89/103482 objects misplaced (0.086%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:34:07.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:07.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[107556]: osdmap e235: 8 total, 7 up, 8 in 2024-09-13T14:34:07.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[103001]: pgmap v632: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 89/103482 objects misplaced (0.086%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:34:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[103001]: osdmap e235: 8 total, 7 up, 8 in 2024-09-13T14:34:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:07.531 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:07 smithi017 podman[210559]: 2024-09-13 14:34:07.246172018 +0000 UTC m=+6.090574596 container died bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:07.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:07.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:07.682 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:07.715 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:07 smithi189 ceph-mon[90992]: pgmap v632: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 89/103482 objects misplaced (0.086%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:34:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:07 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:07 smithi189 ceph-mon[90992]: osdmap e235: 8 total, 7 up, 8 in 2024-09-13T14:34:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:07.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:07.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:07.939 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:07 smithi017 podman[210559]: 2024-09-13 14:34:07.719794178 +0000 UTC m=+6.564196716 container cleanup bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:34:07.940 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:07 smithi017 bash[210559]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:34:07.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:07.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:07.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:08.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:08.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:08.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:07 smithi017 podman[211244]: 2024-09-13 14:34:07.939011482 +0000 UTC m=+0.824136202 container remove bc4e38b82905c0469e270938b57add9a2d730ecc1536ce0388825ad0d29d1286 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:08.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:08.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:08.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:08.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:08.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:34:08.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:08.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:08.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:08.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:08.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.831 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:08.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:08.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[103001]: osdmap e236: 8 total, 7 up, 8 in 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[103001]: 6.19 continuing backfill to osd.3 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[103001]: 6.19 continuing backfill to osd.5 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[103001]: 6.18 continuing backfill to osd.0 from (182'4400,234'12741] 6:1f067de8:::benchmark_data_smithi017_185083_object104969:head to 234'12741 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[103001]: osdmap e237: 8 total, 7 up, 8 in 2024-09-13T14:34:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[107556]: osdmap e236: 8 total, 7 up, 8 in 2024-09-13T14:34:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[107556]: 6.19 continuing backfill to osd.3 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[107556]: 6.19 continuing backfill to osd.5 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[107556]: 6.18 continuing backfill to osd.0 from (182'4400,234'12741] 6:1f067de8:::benchmark_data_smithi017_185083_object104969:head to 234'12741 2024-09-13T14:34:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:08 smithi017 ceph-mon[107556]: osdmap e237: 8 total, 7 up, 8 in 2024-09-13T14:34:08.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:08.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:08 smithi189 ceph-mon[90992]: osdmap e236: 8 total, 7 up, 8 in 2024-09-13T14:34:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:08 smithi189 ceph-mon[90992]: 6.19 continuing backfill to osd.3 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:08 smithi189 ceph-mon[90992]: 6.19 continuing backfill to osd.5 from (179'2700,234'12686] MIN to 234'12686 2024-09-13T14:34:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:08 smithi189 ceph-mon[90992]: 6.18 continuing backfill to osd.0 from (182'4400,234'12741] 6:1f067de8:::benchmark_data_smithi017_185083_object104969:head to 234'12741 2024-09-13T14:34:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:08 smithi189 ceph-mon[90992]: osdmap e237: 8 total, 7 up, 8 in 2024-09-13T14:34:09.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:09.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:09.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:09.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:09.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:09.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:09.434 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:09.453 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:09.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:09.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:09.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:09.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[107556]: pgmap v635: 109 pgs: 2 active+undersized, 11 stale+active+clean, 3 active+remapped+backfilling, 3 active+undersized+degraded, 90 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 942/101076 objects degraded (0.932%); 3215/101076 objects misplaced (3.181%); 14 MiB/s, 221 objects/s recovering 2024-09-13T14:34:09.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 942/101076 objects degraded (0.932%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:09.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[107556]: osdmap e238: 8 total, 7 up, 8 in 2024-09-13T14:34:09.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[103001]: pgmap v635: 109 pgs: 2 active+undersized, 11 stale+active+clean, 3 active+remapped+backfilling, 3 active+undersized+degraded, 90 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 942/101076 objects degraded (0.932%); 3215/101076 objects misplaced (3.181%); 14 MiB/s, 221 objects/s recovering 2024-09-13T14:34:09.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 942/101076 objects degraded (0.932%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:09.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:09 smithi017 ceph-mon[103001]: osdmap e238: 8 total, 7 up, 8 in 2024-09-13T14:34:09.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:09.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:09.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:09.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:09.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:09.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:09 smithi189 ceph-mon[90992]: pgmap v635: 109 pgs: 2 active+undersized, 11 stale+active+clean, 3 active+remapped+backfilling, 3 active+undersized+degraded, 90 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 942/101076 objects degraded (0.932%); 3215/101076 objects misplaced (3.181%); 14 MiB/s, 221 objects/s recovering 2024-09-13T14:34:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:09 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 942/101076 objects degraded (0.932%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:09 smithi189 ceph-mon[90992]: osdmap e238: 8 total, 7 up, 8 in 2024-09-13T14:34:10.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:10.038 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:34:10.038 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:34:10.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:10.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:10.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:10.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:10.300 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:10.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:10.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:10.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:10.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:10.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:10.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:10.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:10.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:10.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:10.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:10.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:10.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:10.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:11.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:11.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:11.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:11.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:11.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:11.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:11.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:11.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:11.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:11 smithi017 ceph-mon[103001]: pgmap v638: 109 pgs: 24 active+undersized, 3 stale+active+clean, 2 active+remapped+backfilling, 11 active+undersized+degraded, 69 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 772 op/s; 8888/94332 objects degraded (9.422%); 3126/94332 objects misplaced (3.314%); 16 MiB/s, 259 objects/s recovering 2024-09-13T14:34:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:11 smithi017 ceph-mon[107556]: pgmap v638: 109 pgs: 24 active+undersized, 3 stale+active+clean, 2 active+remapped+backfilling, 11 active+undersized+degraded, 69 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 772 op/s; 8888/94332 objects degraded (9.422%); 3126/94332 objects misplaced (3.314%); 16 MiB/s, 259 objects/s recovering 2024-09-13T14:34:11.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:11.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:11.897 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:11 smithi189 ceph-mon[90992]: pgmap v638: 109 pgs: 24 active+undersized, 3 stale+active+clean, 2 active+remapped+backfilling, 11 active+undersized+degraded, 69 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 772 op/s; 8888/94332 objects degraded (9.422%); 3126/94332 objects misplaced (3.314%); 16 MiB/s, 259 objects/s recovering 2024-09-13T14:34:11.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:11.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:11.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:12.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:12.100 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:12.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:12.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:12.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:12.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:12.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:12.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:12.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:12.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:12.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:12.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:12.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:12.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:34:12.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:12.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:12.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:13.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:13.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:13.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:13.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:13.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:13.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:13.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:13.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[103001]: pgmap v639: 109 pgs: 28 active+undersized, 2 active+remapped+backfilling, 11 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 13 KiB/s rd, 27 KiB/s wr, 583 op/s; 8888/93207 objects degraded (9.536%); 3126/93207 objects misplaced (3.354%); 993 KiB/s, 15 objects/s recovering 2024-09-13T14:34:13.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:13.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:13.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[107556]: pgmap v639: 109 pgs: 28 active+undersized, 2 active+remapped+backfilling, 11 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 13 KiB/s rd, 27 KiB/s wr, 583 op/s; 8888/93207 objects degraded (9.536%); 3126/93207 objects misplaced (3.354%); 993 KiB/s, 15 objects/s recovering 2024-09-13T14:34:13.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:13.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:13.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:13 smithi189 ceph-mon[90992]: pgmap v639: 109 pgs: 28 active+undersized, 2 active+remapped+backfilling, 11 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 13 KiB/s rd, 27 KiB/s wr, 583 op/s; 8888/93207 objects degraded (9.536%); 3126/93207 objects misplaced (3.354%); 993 KiB/s, 15 objects/s recovering 2024-09-13T14:34:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:13.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:13.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:13.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:13.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:13.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:13.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:13.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:13.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:13.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:13.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:14.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:14.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:14.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:14.217 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:34:14.217 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:34:14.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:14.224 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-13T14:34:14.256 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:34:14.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:34:14.294 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:34:14.294 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:34:14.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:14 smithi017 ceph-mon[103001]: osdmap e239: 8 total, 7 up, 8 in 2024-09-13T14:34:14.295 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:14 smithi017 ceph-mon[107556]: osdmap e239: 8 total, 7 up, 8 in 2024-09-13T14:34:14.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:14.325 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:34:14.326 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:34:14.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:14.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:34:14.366 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:34:14.399 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:34:14.399 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:34:14.433 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:34:14.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:34:14.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:14.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:14 smithi189 ceph-mon[90992]: osdmap e239: 8 total, 7 up, 8 in 2024-09-13T14:34:14.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:14.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:14.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:14.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:14.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:14.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:14.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:14.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:15.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:15.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:15.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:15.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:15.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:15.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:15.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[103001]: pgmap v641: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 1.10k op/s; 7633/81297 objects degraded (9.389%); 1656/81297 objects misplaced (2.037%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:34:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 7633/81297 objects degraded (9.389%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[103001]: osdmap e240: 8 total, 7 up, 8 in 2024-09-13T14:34:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2083013862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[107556]: pgmap v641: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 1.10k op/s; 7633/81297 objects degraded (9.389%); 1656/81297 objects misplaced (2.037%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:34:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 7633/81297 objects degraded (9.389%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[107556]: osdmap e240: 8 total, 7 up, 8 in 2024-09-13T14:34:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2083013862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:15 smithi189 ceph-mon[90992]: pgmap v641: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 1.10k op/s; 7633/81297 objects degraded (9.389%); 1656/81297 objects misplaced (2.037%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:34:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:15 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 7633/81297 objects degraded (9.389%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:15.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:15 smithi189 ceph-mon[90992]: osdmap e240: 8 total, 7 up, 8 in 2024-09-13T14:34:15.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2083013862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:15.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:15.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:15.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:15.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:15.809 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:15.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:15.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:15.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:15.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:16.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:16.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:16.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:16.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:16.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:16.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:16.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:16.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:16.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:16.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:16.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:16.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:16.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:16.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:16.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:16.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:17.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:17.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:17.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:17.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:17.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:17.337 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:17.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:17 smithi017 ceph-mon[103001]: pgmap v643: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7255/77499 objects degraded (9.361%); 1656/77499 objects misplaced (2.137%); 9.9 MiB/s, 158 objects/s recovering 2024-09-13T14:34:17.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:17 smithi017 ceph-mon[107556]: pgmap v643: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7255/77499 objects degraded (9.361%); 1656/77499 objects misplaced (2.137%); 9.9 MiB/s, 158 objects/s recovering 2024-09-13T14:34:17.350 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:17 smithi017 podman[212225]: 2024-09-13 14:34:16.907294297 +0000 UTC m=+2.668952822 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:17.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:17.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:17.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:17.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:17 smithi189 ceph-mon[90992]: pgmap v643: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7255/77499 objects degraded (9.361%); 1656/77499 objects misplaced (2.137%); 9.9 MiB/s, 158 objects/s recovering 2024-09-13T14:34:17.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:17.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:17] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-09-13T14:34:17.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:17.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:17.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:17.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:17.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:18.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:18.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:18.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:18.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:18.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:18.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:18.419 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:18.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:18.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:18.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:18.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:18.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:18.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:18.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:18.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:19.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:19.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:34:19.150 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:34:19.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:19.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:19.194 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:19.208 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:19.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:19.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:19.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:19 smithi017 ceph-mon[103001]: pgmap v644: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7074/73287 objects degraded (9.652%); 1483/73287 objects misplaced (2.024%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:34:19.358 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:19 smithi017 ceph-mon[107556]: pgmap v644: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7074/73287 objects degraded (9.652%); 1483/73287 objects misplaced (2.024%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:34:19.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:19.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:19.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:19.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:19.696 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:19 smithi017 podman[212225]: 2024-09-13 14:34:19.39720214 +0000 UTC m=+5.158860655 container create d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:19.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:19.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:19 smithi189 ceph-mon[90992]: pgmap v644: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 7074/73287 objects degraded (9.652%); 1483/73287 objects misplaced (2.024%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:34:19.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:19.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:19.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:19.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:19.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:19 smithi017 podman[212225]: 2024-09-13 14:34:19.944816014 +0000 UTC m=+5.706474539 container init d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:34:19.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:20.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:20.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:20.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:20.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:20.259 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:19 smithi017 podman[212225]: 2024-09-13 14:34:19.953007442 +0000 UTC m=+5.714665950 container start d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:20.260 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:20 smithi017 podman[212225]: 2024-09-13 14:34:20.178104225 +0000 UTC m=+5.939762743 container attach d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T14:34:20.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:20.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:20.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:20 smithi189 ceph-mon[90992]: pgmap v645: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 5949/64254 objects degraded (9.259%); 1303/64254 objects misplaced (2.028%); 8.4 MiB/s, 135 objects/s recovering 2024-09-13T14:34:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:20.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:20.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:20.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[103001]: pgmap v645: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 5949/64254 objects degraded (9.259%); 1303/64254 objects misplaced (2.028%); 8.4 MiB/s, 135 objects/s recovering 2024-09-13T14:34:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[107556]: pgmap v645: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.29k op/s; 5949/64254 objects degraded (9.259%); 1303/64254 objects misplaced (2.028%); 8.4 MiB/s, 135 objects/s recovering 2024-09-13T14:34:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:20.715 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:20.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:20.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:20.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:20.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:21.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:21.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:21.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:34:21.189 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:20 smithi017 podman[212225]: 2024-09-13 14:34:20.893731096 +0000 UTC m=+6.655389609 container died d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:34:21.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:21.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:21.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:21 smithi017 ceph-mon[107556]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T14:34:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:21 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5949/64254 objects degraded (9.259%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:21 smithi017 ceph-mon[103001]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T14:34:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:21 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5949/64254 objects degraded (9.259%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:21.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:21.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:21.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:21.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:21.737 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:21 smithi189 ceph-mon[90992]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T14:34:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:21 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5949/64254 objects degraded (9.259%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:21.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:21.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:21.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:21.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:22.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:22.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:22.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:22.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:21 smithi017 podman[212884]: 2024-09-13 14:34:21.774624881 +0000 UTC m=+1.503899698 container remove d9cd323bceed81119a2431d1de829af756087380306373b6fd7cfbe7f1d361d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T14:34:22.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T14:34:22.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:22.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:22.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:22.327 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:34:22.327 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T14:34:22.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:22.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:22.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:22.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:22.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:22 smithi017 ceph-mon[103001]: pgmap v646: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 5540/59925 objects degraded (9.245%); 1303/59925 objects misplaced (2.174%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:22 smithi017 ceph-mon[107556]: pgmap v646: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 5540/59925 objects degraded (9.245%); 1303/59925 objects misplaced (2.174%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:22.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:22 smithi189 ceph-mon[90992]: pgmap v646: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 5540/59925 objects degraded (9.245%); 1303/59925 objects misplaced (2.174%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:22.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:22.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:22.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:22.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.006 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:23.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:23.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:23.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:23.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:23.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:23.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:23.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:23.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:23.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:23.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:23.684 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:23.786 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:23.818 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:23.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:23.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:23.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:23.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:24.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:24.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:24.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:24.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:24.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:24.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:24.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:24.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:24.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:24.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:24.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:24 smithi017 ceph-mon[103001]: pgmap v647: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 927 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s; 4034/44373 objects degraded (9.091%); 853/44373 objects misplaced (1.922%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:24 smithi017 ceph-mon[107556]: pgmap v647: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 927 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s; 4034/44373 objects degraded (9.091%); 853/44373 objects misplaced (1.922%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:24.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:24 smithi189 ceph-mon[90992]: pgmap v647: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 927 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s; 4034/44373 objects degraded (9.091%); 853/44373 objects misplaced (1.922%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:24.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:24.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:24.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:24.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:25.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:25.019 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:34:25.019 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:34:25.019 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:34:25.019 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:34:25.019 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:34:25.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:25.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:25.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:25.272 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:25.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:25.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:25.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:25.452 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:34:25.471 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:34:25.471 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:34:25.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:25.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:25.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:25.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:34:25.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:25.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:25.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:25.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:25.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:26.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:26.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:26.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:26.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:26.363 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:26.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:26.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:26.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:26.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:26.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[103001]: pgmap v648: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 840 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 3606/40203 objects degraded (8.969%); 853/40203 objects misplaced (2.122%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 3606/40203 objects degraded (8.969%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[103001]: osdmap e241: 8 total, 7 up, 8 in 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[107556]: pgmap v648: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 840 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 3606/40203 objects degraded (8.969%); 853/40203 objects misplaced (2.122%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 3606/40203 objects degraded (8.969%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:34:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:26 smithi017 ceph-mon[107556]: osdmap e241: 8 total, 7 up, 8 in 2024-09-13T14:34:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:26 smithi189 ceph-mon[90992]: pgmap v648: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 840 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 3606/40203 objects degraded (8.969%); 853/40203 objects misplaced (2.122%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:26 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:26 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 3606/40203 objects degraded (8.969%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2073029228' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:34:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:26 smithi189 ceph-mon[90992]: osdmap e241: 8 total, 7 up, 8 in 2024-09-13T14:34:26.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:26.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:26.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:26.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:26.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:27.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:27.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:27.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:27.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:27.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:27.243 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:34:27.244 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:34:27.244 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-13T14:34:27.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:34:27.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:34:27.328 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2024-09-13T14:34:27.329 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2024-09-13T14:34:27.329 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2024-09-13T14:34:27.329 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:34:27.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:34:27.331 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:34:27.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl start ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1 2024-09-13T14:34:27.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:27.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:27.367 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:34:27.367 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:34:27.398 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:34:27.398 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:34:27.420 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:27] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-09-13T14:34:27.428 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:34:27.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:34:27.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:34:27.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:34:27.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:27.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:27.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:27.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:27.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:27 smithi017 systemd[1]: Starting Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:34:27.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:27.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:27.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:34:27.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:27.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:27.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:27.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:28.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:28.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:28.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:28.224 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:28.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.359 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: 6.12 scrub starts 2024-09-13T14:34:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: 6.12 scrub ok 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1198320604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[107556]: pgmap v650: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 803 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s; 3606/38397 objects degraded (9.391%); 853/38397 objects misplaced (2.222%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: 6.12 scrub starts 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: 6.12 scrub ok 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1198320604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:28 smithi017 ceph-mon[103001]: pgmap v650: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 803 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s; 3606/38397 objects degraded (9.391%); 853/38397 objects misplaced (2.222%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:28.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:28.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: 6.12 scrub starts 2024-09-13T14:34:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: 6.12 scrub ok 2024-09-13T14:34:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1198320604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:28 smithi189 ceph-mon[90992]: pgmap v650: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 803 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.23k op/s; 3606/38397 objects degraded (9.391%); 853/38397 objects misplaced (2.222%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:28.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:28.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:34:28.643 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:34:28.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:28.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:28.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:28.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:34:28.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:28.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:28.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:28 smithi017 podman[213852]: 2024-09-13 14:34:28.67685009 +0000 UTC m=+0.043306577 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:28.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:28 smithi017 podman[213852]: 2024-09-13 14:34:28.810874887 +0000 UTC m=+0.177331365 container create 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T14:34:29.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:29.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:29.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:29.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.195 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:29.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 podman[213852]: 2024-09-13 14:34:29.124883676 +0000 UTC m=+0.491340164 container init 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T14:34:29.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 podman[213852]: 2024-09-13 14:34:29.135443691 +0000 UTC m=+0.501900166 container start 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T14:34:29.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:29.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:29.328 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:29.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:29.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:29.529 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 podman[213852]: 2024-09-13 14:34:29.247749465 +0000 UTC m=+0.614205942 container attach 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:34:29.530 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:34:29.543 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:29.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:29.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:29.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:29.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:29.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:29.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:29.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:29.952 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:34:29.952 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:34:29.952 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:34:29.952 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:34:29.952 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate[213911]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 bash[213852]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:34:29.953 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:29 smithi017 podman[213852]: 2024-09-13 14:34:29.691603507 +0000 UTC m=+1.058059993 container died 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:29.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:29.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:30.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:30.140 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:30.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:30.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:30.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:30 smithi189 ceph-mon[90992]: 6.17 scrub starts 2024-09-13T14:34:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:30 smithi189 ceph-mon[90992]: pgmap v651: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 536 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 2282/25602 objects degraded (8.913%); 449/25602 objects misplaced (1.754%) 2024-09-13T14:34:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:30 smithi189 ceph-mon[90992]: 6.17 scrub ok 2024-09-13T14:34:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:30.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:30.561 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:30.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:34:30.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:30.699 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[103001]: 6.17 scrub starts 2024-09-13T14:34:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[103001]: pgmap v651: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 536 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 2282/25602 objects degraded (8.913%); 449/25602 objects misplaced (1.754%) 2024-09-13T14:34:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[103001]: 6.17 scrub ok 2024-09-13T14:34:30.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[107556]: 6.17 scrub starts 2024-09-13T14:34:30.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[107556]: pgmap v651: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 536 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 2282/25602 objects degraded (8.913%); 449/25602 objects misplaced (1.754%) 2024-09-13T14:34:30.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:30 smithi017 ceph-mon[107556]: 6.17 scrub ok 2024-09-13T14:34:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.703 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:30 smithi017 podman[214135]: 2024-09-13 14:34:30.457151515 +0000 UTC m=+0.809990958 container remove 6e8a2714da924499a6b4760bc85c7488fb29bf618a8776bab086b70be06a2bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:34:30.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:30.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:30.839 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:30.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:30.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:30.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:31.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:31.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:31.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:31.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:31.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:31.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:31.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:31.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:31.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:31.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:31.666 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:31 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 1897/21429 objects degraded (8.852%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:31.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:31 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 1897/21429 objects degraded (8.852%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:31.682 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:31 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 1897/21429 objects degraded (8.852%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:31.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:31.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:31.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:31.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:31.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:32.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:32.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:32.125 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:32.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:32.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:32.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:32.363 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:32.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:32.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:32.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:32 smithi017 ceph-mon[103001]: pgmap v652: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 1897/21429 objects degraded (8.852%); 449/21429 objects misplaced (2.095%) 2024-09-13T14:34:32.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:32 smithi017 ceph-mon[107556]: pgmap v652: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 1897/21429 objects degraded (8.852%); 449/21429 objects misplaced (2.095%) 2024-09-13T14:34:32.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:32.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:32.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:32 smithi189 ceph-mon[90992]: pgmap v652: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 1897/21429 objects degraded (8.852%); 449/21429 objects misplaced (2.095%) 2024-09-13T14:34:32.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:32.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:32.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:32.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:32.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:32.952 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:34:32.952 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:34:33.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:33.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:33.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:33.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:33.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:33.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:33.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:33.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:33.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:33.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:33.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:33.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.772 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:33.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:33.831 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 124609 objects 2024-09-13T14:34:33.832 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :147.133 2024-09-13T14:34:33.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:33.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:33.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:33.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:34.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:34.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:34.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:34.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:34.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:34.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:34.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:34.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:34.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:34.500 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:34 smithi017 podman[214470]: 2024-09-13 14:34:34.133804167 +0000 UTC m=+1.471144307 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:34.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:34.571 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-13T14:34:34.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-13T14:34:34.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:34.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:34.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:34 smithi189 ceph-mon[90992]: pgmap v653: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 140 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 537/6570 objects degraded (8.174%); 77/6570 objects misplaced (1.172%) 2024-09-13T14:34:34.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:34.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:34.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:34.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:34 smithi017 ceph-mon[103001]: pgmap v653: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 140 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 537/6570 objects degraded (8.174%); 77/6570 objects misplaced (1.172%) 2024-09-13T14:34:34.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:34 smithi017 ceph-mon[107556]: pgmap v653: 109 pgs: 2 active+clean+remapped, 28 active+undersized, 11 active+undersized+degraded, 68 active+clean; 140 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 537/6570 objects degraded (8.174%); 77/6570 objects misplaced (1.172%) 2024-09-13T14:34:34.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:34.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:34.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:34.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:35.021 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:35.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:35.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:35.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:35.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:35.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:35.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:35.508 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:34:35.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:35.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:35.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:35.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1441582974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:35 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:35.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:35.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:35.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:35.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1441582974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:35 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1441582974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:35 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:34:36.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:36.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:36.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:36.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:36.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:36.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:36.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:36.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:34:36.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:36.407 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:36.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:36.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:36.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[103001]: pgmap v654: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230/3198 objects degraded (7.192%); 77/3198 objects misplaced (2.408%) 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[103001]: osdmap e242: 8 total, 7 up, 8 in 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[107556]: pgmap v654: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230/3198 objects degraded (7.192%); 77/3198 objects misplaced (2.408%) 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:36.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:36 smithi017 ceph-mon[107556]: osdmap e242: 8 total, 7 up, 8 in 2024-09-13T14:34:36.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:36.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:36 smithi189 ceph-mon[90992]: pgmap v654: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 69 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 230/3198 objects degraded (7.192%); 77/3198 objects misplaced (2.408%) 2024-09-13T14:34:36.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:36 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:34:36.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:36 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:34:36.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:36 smithi189 ceph-mon[90992]: osdmap e242: 8 total, 7 up, 8 in 2024-09-13T14:34:36.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:36.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:36.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:36.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:36.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:36.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:36.914 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:36 smithi017 podman[214470]: 2024-09-13 14:34:36.703162336 +0000 UTC m=+4.040502508 container create a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T14:34:37.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:37.031 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:34:37.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:34:37.031 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-13T14:34:37.032 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:34:37.033 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:34:37.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:37.049 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:34:37.049 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:34:37.065 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:34:37.065 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:34:37.081 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:34:37.081 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:34:37.098 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:34:37.098 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:34:37.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:34:37.115 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:34:37.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:37.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:37.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:37.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:37.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:34:37.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:37.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:37] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-09-13T14:34:37.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:37 smithi017 podman[214470]: 2024-09-13 14:34:37.173728762 +0000 UTC m=+4.511068893 container init a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:34:37.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:37 smithi017 podman[214470]: 2024-09-13 14:34:37.179714598 +0000 UTC m=+4.517054729 container start a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T14:34:37.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:37 smithi017 ceph-osd[215218]: -- 172.21.15.17:0/3936702630 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d400e93a40 con 0x55d401c60000 2024-09-13T14:34:37.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:37 smithi017 bash[214470]: a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 2024-09-13T14:34:37.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:37 smithi017 systemd[1]: Started Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:34:37.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:37.539 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:37.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:37.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:37.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:37.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:37.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:37.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:37.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2830379367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2830379367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:38.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2830379367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:38.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:38.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:38.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:38.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:38.270 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:38.271 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:38.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:38.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:38.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:38.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:38.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:38.578 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:38.680 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:38 smithi017 ceph-mon[103001]: pgmap v656: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 52 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 230/2385 objects degraded (9.644%); 77/2385 objects misplaced (3.229%) 2024-09-13T14:34:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:38 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 230/2385 objects degraded (9.644%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:38 smithi017 ceph-mon[107556]: pgmap v656: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 52 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 230/2385 objects degraded (9.644%); 77/2385 objects misplaced (3.229%) 2024-09-13T14:34:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:38 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 230/2385 objects degraded (9.644%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:38.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:38.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:34:38.822 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:34:38.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:38.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:38.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:38.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:38.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:38.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:38 smithi189 ceph-mon[90992]: pgmap v656: 109 pgs: 2 active+clean+remapped, 29 active+undersized, 10 active+undersized+degraded, 68 active+clean; 52 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 230/2385 objects degraded (9.644%); 77/2385 objects misplaced (3.229%) 2024-09-13T14:34:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:38 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 230/2385 objects degraded (9.644%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:39.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:39.069 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:39.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:39.086 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:38 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T14:34:38.826+0000 7fcb2eed5640 -1 Falling back to public interface 2024-09-13T14:34:39.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:39.212 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:39.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:39.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:39.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:39.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:39.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:39.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:39.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:39.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:39.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:39.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:39.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:39.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:39.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:39.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:39.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:40.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:34:40.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:40.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:40.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:40.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:40.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:40.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:40.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:40.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:40.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:40.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:40.604 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:40 smithi017 ceph-mon[107556]: pgmap v657: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:40 smithi017 ceph-mon[103001]: pgmap v657: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:40.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:40.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:40.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:34:40.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:40.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:40.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:40.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:40.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:40 smithi189 ceph-mon[90992]: pgmap v657: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:41.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:41.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:41.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:41.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:41.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:41.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:41.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:41.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:41.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:41.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:41.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:41.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:41.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:41.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:41.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:41.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:41.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:41.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:42.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:42.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:42.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:42.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:42.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:42.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:42.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:42.416 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:42.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:42.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:42.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[103001]: pgmap v658: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:42.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:42.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:42.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[107556]: pgmap v658: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:42.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:42.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:42.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:42.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:42 smithi189 ceph-mon[90992]: pgmap v658: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:42.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:42.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:42.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:42.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:42.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:43.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:43.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:43.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:43.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:43.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:43.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:43.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:43.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:43.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:43.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:43.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:43.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:43.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:43.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:43.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:43.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:43.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:43.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:43.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:43.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:44.032 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:44.058 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-13T14:34:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator fragmentation block 2024-09-13T14:34:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:admin_socket: invalid command 2024-09-13T14:34:44.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:44.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:44.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:44.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:44.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:44.413 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:44.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:44.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:44.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:44.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:44.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:44.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-mon[103001]: pgmap v659: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:34:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-mon[107556]: pgmap v659: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:34:44.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T14:34:44.473+0000 7fcb2eed5640 -1 osd.1 234 log_to_monitors true 2024-09-13T14:34:44.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 14:34:44 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T14:34:44.548+0000 7fcb22c77640 -1 osd.1 243 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:34:44.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:44.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:44.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:44.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:44.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:44 smithi189 ceph-mon[90992]: pgmap v659: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:44 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:34:45.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:45.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:45.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:45.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:45.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:45.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:45.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:45.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:45.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:45.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:45.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:45.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:45.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:45.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:45.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:34:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[103001]: osdmap e243: 8 total, 7 up, 8 in 2024-09-13T14:34:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[103001]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:34:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:34:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[107556]: osdmap e243: 8 total, 7 up, 8 in 2024-09-13T14:34:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:45 smithi017 ceph-mon[107556]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:34:45.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:45.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:45 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:34:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:45 smithi189 ceph-mon[90992]: osdmap e243: 8 total, 7 up, 8 in 2024-09-13T14:34:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:45 smithi189 ceph-mon[90992]: from='osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-13T14:34:46.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:46.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:46.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:46.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:46.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:46.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:46.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:46.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:46.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:46.542 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' removed 2024-09-13T14:34:46.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:46.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: pgmap v661: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329] boot 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1046993930' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: pgmap v661: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329] boot 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1046993930' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:46 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:46.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:46.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:46.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:46.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:46.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:46.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:46.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: pgmap v661: 109 pgs: 2 active+clean+remapped, 34 active+undersized, 5 active+undersized+degraded, 68 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 6/84 objects degraded (7.143%) 2024-09-13T14:34:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: osd.1 [v2:172.21.15.17:6810/176828329,v1:172.21.15.17:6811/176828329] boot 2024-09-13T14:34:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:34:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:34:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1046993930' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:46 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:34:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:34:47.016 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2024-09-13T14:34:47.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:47.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:47.213 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:47.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:47.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:47.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:47.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:47.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:47.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:47.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:47.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:47.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:47] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.51.0" 2024-09-13T14:34:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:47 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:34:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:47 smithi017 ceph-mon[103001]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:34:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:47 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:34:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:47 smithi017 ceph-mon[107556]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:34:47.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:47.735 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:47.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:47.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:47.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:47.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:47.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:47.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:47 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:34:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:47 smithi189 ceph-mon[90992]: osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:34:48.082 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:48.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:48.085 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:34:48.085 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:34:48.086 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-13T14:34:48.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-13T14:34:48.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:48.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:48.229 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:48.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:48.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:48.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:48.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:48.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[103001]: pgmap v664: 83 pgs: 1 peering, 27 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:48.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:34:48.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[103001]: 1.3 scrub ok 2024-09-13T14:34:48.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[107556]: pgmap v664: 83 pgs: 1 peering, 27 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:48.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:34:48.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:48 smithi017 ceph-mon[107556]: 1.3 scrub ok 2024-09-13T14:34:48.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:48.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:48.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:48.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:48.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:48.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:48.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:48.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:48.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:48 smithi189 ceph-mon[90992]: pgmap v664: 83 pgs: 1 peering, 27 active+undersized, 5 active+undersized+degraded, 50 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-09-13T14:34:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:48 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:34:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:48 smithi189 ceph-mon[90992]: 1.3 scrub ok 2024-09-13T14:34:49.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:49.104 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:49.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:49.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:49.306 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:34:49.306 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:34:49.306 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:34:49.306 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:34:49.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:49.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:49.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:49.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:49.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:49.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:49.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:49.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:49.834 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:49.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:49 smithi017 ceph-mon[107556]: pgmap v665: 83 pgs: 1 peering, 15 active+undersized, 1 active+undersized+degraded, 66 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:34:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:34:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:49 smithi017 ceph-mon[103001]: pgmap v665: 83 pgs: 1 peering, 15 active+undersized, 1 active+undersized+degraded, 66 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:34:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:34:49.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:49.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:49.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:49 smithi189 ceph-mon[90992]: pgmap v665: 83 pgs: 1 peering, 15 active+undersized, 1 active+undersized+degraded, 66 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-09-13T14:34:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:34:50.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:50.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:50.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:50.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:50.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:50.337 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:50.363 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:50.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:50.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:34:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:34:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:50.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:50.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:50.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:50.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:50.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:50.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:50.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:50.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:50.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:51.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:51.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:51.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:51.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:51.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:51.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:51.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:51.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:51.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:51.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:51.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:51.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:51 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:34:51.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:51 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:34:51.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:51.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:51 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-09-13T14:34:51.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:51.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:51.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:51.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:51.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:34:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:52.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:34:52.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:52.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:52.203 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:34:52.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:34:52.209 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:34:52.209 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-13T14:34:52.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:52.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:52.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:34:52.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:34:52.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:52.262 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:34:52.262 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:34:52.279 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:34:52.280 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:34:52.296 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:34:52.296 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:34:52.312 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:34:52.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:34:52.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:34:52.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:34:52.363 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:52.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:52.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:52.504 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:34:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:52.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:52.619 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:52.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:52.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:52 smithi189 ceph-mon[90992]: pgmap v666: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1797726303' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:52 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1073492915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:52.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:52.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:52.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:52.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:52.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:52.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[103001]: pgmap v666: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:52.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1797726303' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1073492915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:52.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[107556]: pgmap v666: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:52.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1797726303' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:34:52.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1073492915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:34:53.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:53.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:53.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:53.151 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:53.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:53.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:53.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:53.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:53.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:53.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:53.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[103001]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:34:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:53 smithi017 ceph-mon[107556]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:34:53.659 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:53 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:34:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:53 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:34:53.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:53 smithi189 ceph-mon[90992]: osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:34:53.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:53.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:53.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:53.913 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:53.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:54.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:54.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:54.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:54.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:54.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:54.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:54.374 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:54.398 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:54.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:54.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:54.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:54.648 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:54.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:54.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:54 smithi017 ceph-mon[103001]: pgmap v668: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:54.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:54 smithi017 ceph-mon[107556]: pgmap v668: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:54.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:54 smithi189 ceph-mon[90992]: pgmap v668: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:54.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:54.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:54.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:54.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:54.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:54.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:54.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:55.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:55.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:55.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:55.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:55.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:55.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:55.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:55.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:34:55.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:55.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:55.530 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:55.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:55.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:55.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:55.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:55.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:55.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:55.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:55.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:56.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:56.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:56.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:56.186 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:56.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:56.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:56.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:56.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:34:56.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:56.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:56.510 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' created 2024-09-13T14:34:56.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:56.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:56.697 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:56 smithi017 ceph-mon[103001]: pgmap v669: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:34:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:56 smithi017 ceph-mon[107556]: pgmap v669: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:34:56.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:56 smithi189 ceph-mon[90992]: pgmap v669: 83 pgs: 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:34:56.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:56.848 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:56.950 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:56.975 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:56.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:57.077 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:57.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:57.203 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:57.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:57.330 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:57.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-13T14:34:57.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:57.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:57.481 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:57.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:57.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:57.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:34:57] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:34:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[103001]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[107556]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:57.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:57.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:57.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:57.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:57.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:57.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:57.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:57.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2500122537' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:34:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:57 smithi189 ceph-mon[90992]: osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:34:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:34:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:58.081 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:58.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:58.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:58.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:58.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:58.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:58.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:58.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:34:58.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:34:58.587 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:58.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[103001]: pgmap v671: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[103001]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:34:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[103001]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:34:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[107556]: pgmap v671: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[107556]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:34:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:34:58 smithi017 ceph-mon[107556]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:34:58.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:58.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:58.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:34:58.856 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:58.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:34:58.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:34:58.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:58.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:58.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:34:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:58 smithi189 ceph-mon[90992]: pgmap v671: 99 pgs: 16 unknown, 83 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:58 smithi189 ceph-mon[90992]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:34:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:58 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:34:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:34:58 smithi189 ceph-mon[90992]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:34:59.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:59.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:59.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:59.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:59.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:34:59.372 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:34:59.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:34:59.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:34:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:34:59.535 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:34:59.535 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:34:59.545 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:59.545 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:59.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:59.624 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:59.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:34:59.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:34:59.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:34:59.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:34:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:34:59.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:34:59.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:00.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:00.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:00.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:00.181 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:35:00.181 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:35:00.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:00.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:00.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:00.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:00.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:00.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:00.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:00.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:00.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:00.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:00.777 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:00 smithi189 ceph-mon[90992]: pgmap v674: 99 pgs: 6 unknown, 93 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:00.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:00.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:00 smithi017 ceph-mon[103001]: pgmap v674: 99 pgs: 6 unknown, 93 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:00.863 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:00 smithi017 ceph-mon[107556]: pgmap v674: 99 pgs: 6 unknown, 93 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:00.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:00.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:00.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:35:00.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:01.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:01.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:01.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:01.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:01.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:01.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:01.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:01.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:01.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:01.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:01.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:01.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:01.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:01.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:01.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:01.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:01.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:01.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:01.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:02.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:02.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:02.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:02.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:02.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:02.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:02.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:02.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:02.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:02.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:02.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:02.718 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:02.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:02.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:02.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:02 smithi017 ceph-mon[107556]: pgmap v675: 99 pgs: 2 unknown, 97 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:02 smithi017 ceph-mon[103001]: pgmap v675: 99 pgs: 2 unknown, 97 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:02.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:02.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:02.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:02.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:35:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:02 smithi189 ceph-mon[90992]: pgmap v675: 99 pgs: 2 unknown, 97 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:03.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:03.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:03.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:03.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:03.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:03.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:03.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:03.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:03.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:03.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:03.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:03.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:03.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:03.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:03.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:03.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:04.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:04.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:04.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:04.240 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:04.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:04.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:04.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:04.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:04.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:04.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:04 smithi017 ceph-mon[103001]: pgmap v676: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:04 smithi017 ceph-mon[107556]: pgmap v676: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:04.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:04.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:04.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:04.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:04.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:04.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:04.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:04 smithi189 ceph-mon[90992]: pgmap v676: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:05.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:05.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:05.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:05.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:05.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:05.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:05.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:05.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:05.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:05.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:05.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:05.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:05.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:05.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:05.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:05.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:05.922 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:35:05.922 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:35:05.922 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:35:05.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:35:05.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:35:05.923 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:35:05.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:35:05.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:35:05.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:35:05.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:35:05.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:06.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:06.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:06.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:06.172 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:06.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:06.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:06.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:06.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:06.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:06.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:06.539 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:35:06.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:06.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:06.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:06.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:06.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:06.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:06.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:06.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[103001]: pgmap v677: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2677256949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1676268078' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[107556]: pgmap v677: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2677256949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1676268078' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:06 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:06.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:07.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:06 smithi189 ceph-mon[90992]: pgmap v677: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2677256949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1676268078' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:06 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:07.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:07.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:07.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:07.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:07.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:07.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:07.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:07.570 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:07] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-13T14:35:07.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:07.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:07.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[107556]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:35:07.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:07.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:07.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:07 smithi017 ceph-mon[103001]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:35:07.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:07.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:07.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:07.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:07.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:07.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:07.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:07.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:07.898 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:07.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:07 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:07 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:07 smithi189 ceph-mon[90992]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:35:08.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:08.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:08.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:08.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:08.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:08.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:08.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:08.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:08.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:08.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:08.540 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-13T14:35:08.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:08.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:08.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:08.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:35:08.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:08.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:08.859 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:08 smithi017 ceph-mon[103001]: pgmap v679: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:08.859 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.859 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:08 smithi017 ceph-mon[107556]: pgmap v679: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:08.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:08.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:08.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:08.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:08.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:08 smithi189 ceph-mon[90992]: pgmap v679: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:09.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:09.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:09.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:09.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:09.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:09.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:09.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:35:09.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:09.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:09.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:09.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:09.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:09.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:09.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:09 smithi017 ceph-mon[103001]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:35:09.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:09.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:09 smithi017 ceph-mon[107556]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:35:09.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:09.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:09.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:09.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:09.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:09.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:09.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:09.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/476222514' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:09 smithi189 ceph-mon[90992]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:35:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:10.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:10.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:10.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:10.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:10.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:10.373 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:10.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:10.500 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-13T14:35:10.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:10.503 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:35:10.503 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:35:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:10.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:10.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:35:10.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:35:10.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_220393 2024-09-13T14:35:10.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:10.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:35:10.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:10.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:10.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:10.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:10.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:10.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:10.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:10.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:10 smithi017 ceph-mon[103001]: pgmap v681: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:10.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:10 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:10.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:10 smithi017 ceph-mon[107556]: pgmap v681: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:10.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:10 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:10 smithi189 ceph-mon[90992]: pgmap v681: 99 pgs: 99 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:10 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:11.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:11.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:11.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:11.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:11.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:11.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:11.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:11.490 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:11.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 15 3458 3443 215.173 215.188 0.00372408 0.00462546 2024-09-13T14:35:11.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:11.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:11.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:11.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:11.811 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:35:11.811 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:35:11.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:35:11.811 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198817 6.12 6:4f354c86:::benchmark_data_smithi017_185083_object74205:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.241977+0000", 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564931712999993, 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1803028979999999, 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198817 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.812 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241977+0000", 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241977+0000", 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241977+0000", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.63e-07 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241980+0000", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4880000000000001e-06 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241987+0000", 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6580000000000002e-06 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.241993+0000", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8109999999999999e-06 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242051+0000", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7633000000000002e-05 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242058+0000", 2024-09-13T14:35:11.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5020000000000003e-06 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.415543+0000", 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1734843800000001 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.415651+0000", 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010804200000000001 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.415749+0000", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8323e-05 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.420189+0000", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044400619999999998 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.420376+0000", 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018679800000000001 2024-09-13T14:35:11.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.420432+0000", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6648999999999999e-05 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422111+0000", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001678618 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422172+0000", 2024-09-13T14:35:11.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1229999999999995e-05 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422280+0000", 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107241 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198822 6.12 6:4d3efc9f:::benchmark_data_smithi017_185083_object74210:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242011+0000", 2024-09-13T14:35:11.819 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564897860000002, 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1841849280000001, 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198822 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242011+0000", 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242011+0000", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242011+0000", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9999999999999998e-07 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242014+0000", 2024-09-13T14:35:11.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5179999999999999e-06 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242021+0000", 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0940000000000002e-06 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242026+0000", 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3290000000000001e-06 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242083+0000", 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7076e-05 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242090+0000", 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3980000000000002e-06 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.417975+0000", 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1758846780000001 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.418153+0000", 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177567 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.418288+0000", 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134895 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.420390+0000", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021024770000000002 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422331+0000", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019408540000000001 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.425967+0000", 2024-09-13T14:35:11.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036362790000000001 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426018+0000", 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0957e-05 2024-09-13T14:35:11.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426078+0000", 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0226999999999998e-05 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426195+0000", 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011707899999999999 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198910 6.12 6:499b370a:::benchmark_data_smithi017_185083_object74298:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242046+0000", 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564862074999994, 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1844561730000001, 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198910 2024-09-13T14:35:11.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242046+0000", 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242046+0000", 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242047+0000", 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7299999999999995e-07 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242050+0000", 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.729e-06 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242057+0000", 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2649999999999999e-06 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242066+0000", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.867e-06 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242117+0000", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1384999999999997e-05 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242125+0000", 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4660000000000002e-06 2024-09-13T14:35:11.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422361+0000", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1802360809999999 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422473+0000", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112013 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.422589+0000", 2024-09-13T14:35:11.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116628 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.425980+0000", 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003391023 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426244+0000", 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026394 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426298+0000", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4080999999999999e-05 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426341+0000", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2635000000000001e-05 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426398+0000", 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6878e-05 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426502+0000", 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104509 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198916 6.12 6:4d299de5:::benchmark_data_smithi017_185083_object74304:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242077+0000", 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564831283999993, 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.195924674, 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.835 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198916 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242077+0000", 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242077+0000", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242078+0000", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4299999999999998e-07 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242080+0000", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0430000000000002e-06 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242086+0000", 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6630000000000004e-06 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242092+0000", 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4559999999999999e-06 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242158+0000", 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6755999999999996e-05 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242162+0000", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9469999999999996e-06 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.424147+0000", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.181984524 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.424263+0000", 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116446 2024-09-13T14:35:11.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.424379+0000", 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115982 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426544+0000", 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002165124 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.430103+0000", 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035581039999999999 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433841+0000", 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00373888 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437833+0000", 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039918929999999998 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437894+0000", 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0519000000000001e-05 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438002+0000", 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010789399999999999 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.842 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198941 6.12 6:4e3eced2:::benchmark_data_smithi017_185083_object74329:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242100+0000", 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564807961, 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.196083526, 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198941 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242100+0000", 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242100+0000", 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242101+0000", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242108+0000", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6640000000000001e-06 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242115+0000", 2024-09-13T14:35:11.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242120+0000", 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4110000000000004e-06 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242191+0000", 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1044999999999995e-05 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242198+0000", 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.674e-06 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426576+0000", 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.184378261 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426690+0000", 2024-09-13T14:35:11.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113282 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.426817+0000", 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127265 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.431909+0000", 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050924990000000003 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433904+0000", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019943420000000001 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437613+0000", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037097760000000001 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438039+0000", 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042606799999999999 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438088+0000", 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8417000000000001e-05 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438184+0000", 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6039999999999995e-05 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.850 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198963 6.12 6:4da4a26c:::benchmark_data_smithi017_185083_object74351:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242128+0000", 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564780128999999, 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1962567900000001, 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198963 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242128+0000", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242128+0000", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242129+0000", 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4400000000000001e-07 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242131+0000", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.956e-06 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242137+0000", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6080000000000001e-06 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242143+0000", 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5489999999999997e-06 2024-09-13T14:35:11.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242233+0000", 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9685000000000006e-05 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242239+0000", 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8850000000000001e-06 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.428425+0000", 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.186185694 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.428534+0000", 2024-09-13T14:35:11.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109399 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.428653+0000", 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118979 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433855+0000", 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005201471 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433953+0000", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8319999999999994e-05 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437651+0000", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036978279999999998 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438298+0000", 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064652700000000004 2024-09-13T14:35:11.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438332+0000", 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4328000000000001e-05 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438385+0000", 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3117e-05 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198970 6.12 6:49daa1ed:::benchmark_data_smithi017_185083_object74358:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242153+0000", 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564754969999996, 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.208717161, 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198970 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242153+0000", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242153+0000", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242154+0000", 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8100000000000003e-07 2024-09-13T14:35:11.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242156+0000", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3489999999999999e-06 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242163+0000", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8770000000000004e-06 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242169+0000", 2024-09-13T14:35:11.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4410000000000001e-06 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242288+0000", 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118961 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242298+0000", 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0046e-05 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.430134+0000", 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1878359380000001 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.430238+0000", 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104703 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.430348+0000", 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110099 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433863+0000", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035142260000000001 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437574+0000", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037113570000000002 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437688+0000", 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011401800000000001 2024-09-13T14:35:11.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.450695+0000", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013006998000000001 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.450757+0000", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2347000000000005e-05 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.450871+0000", 2024-09-13T14:35:11.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011332000000000001 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:198993 6.12 6:4fed75d4:::benchmark_data_smithi017_185083_object74381:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242190+0000", 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564718541000005, 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2088836650000001, 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.866 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 198993 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242190+0000", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242190+0000", 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242190+0000", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2500000000000001e-07 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242192+0000", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1469999999999999e-06 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242200+0000", 2024-09-13T14:35:11.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1169999999999998e-06 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242205+0000", 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6720000000000001e-06 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242350+0000", 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144537 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242358+0000", 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1259999999999998e-06 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.431941+0000", 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1895827560000001 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.432054+0000", 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011312200000000001 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.432174+0000", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119879 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437724+0000", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005550676 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437761+0000", 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6602e-05 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.443101+0000", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053403249999999999 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.450927+0000", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078253079999999996 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.450974+0000", 2024-09-13T14:35:11.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7840999999999998e-05 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.451074+0000", 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9131999999999999e-05 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:199006 6.12 6:4bffbcb0:::benchmark_data_smithi017_185083_object74394:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e207)", 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:35.242229+0000", 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.564679730999998, 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.209175033, 2024-09-13T14:35:11.873 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 199006 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242229+0000", 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242229+0000", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242229+0000", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5600000000000001e-07 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242232+0000", 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.402e-06 2024-09-13T14:35:11.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242239+0000", 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242244+0000", 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.592e-06 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242389+0000", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014489700000000001 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:35.242397+0000", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9000000000000006e-06 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.433985+0000", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1915885099999999 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.434098+0000", 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011252300000000001 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,7", 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.434212+0000", 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114351 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.437797+0000", 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003585146 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.438221+0000", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000423418 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.443146+0000", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049248249999999999 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.451247+0000", 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081008060000000003 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.451302+0000", 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5529000000000003e-05 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:36.451404+0000", 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101629 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.880 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214326 6.18 6:1c49ea0e:::benchmark_data_smithi017_185083_object89714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581388+0000", 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225520200000005, 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183332448, 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214326 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581388+0000", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581388+0000", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581389+0000", 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0200000000000003e-07 2024-09-13T14:35:11.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.883 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581391+0000", 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0559999999999999e-06 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581398+0000", 2024-09-13T14:35:11.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9739999999999996e-06 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581404+0000", 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3439999999999997e-06 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583293+0000", 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018885970000000001 2024-09-13T14:35:11.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583299+0000", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3010000000000003e-06 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.758314+0000", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.175014819 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.758438+0000", 2024-09-13T14:35:11.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124521 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.758723+0000", 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028463000000000002 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762648+0000", 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039248449999999997 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762662+0000", 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4551e-05 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762696+0000", 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3859000000000002e-05 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762736+0000", 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0059999999999999e-05 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764547+0000", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018110400000000001 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764598+0000", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0481e-05 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764721+0000", 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012296799999999999 2024-09-13T14:35:11.889 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214336 6.18 6:1b8245f8:::benchmark_data_smithi017_185083_object89724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581419+0000", 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225489194000005, 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.188184702, 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214336 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581419+0000", 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581419+0000", 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581420+0000", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581422+0000", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2110000000000001e-06 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581430+0000", 2024-09-13T14:35:11.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7500000000000003e-06 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581435+0000", 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7300000000000002e-06 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583323+0000", 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018877500000000001 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583329+0000", 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2639999999999997e-06 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.760448+0000", 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1771189950000001 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.760645+0000", 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019714299999999999 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.761002+0000", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035631900000000002 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762750+0000", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017481440000000001 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762784+0000", 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4150000000000003e-05 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764446+0000", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001662406 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764496+0000", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9122000000000002e-05 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769398+0000", 2024-09-13T14:35:11.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049023269999999997 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769468+0000", 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0261999999999999e-05 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769604+0000", 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135766 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214374 6.18 6:1f2d05a2:::benchmark_data_smithi017_185083_object89762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581444+0000", 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225464587999994, 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1888511639999999, 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214374 2024-09-13T14:35:11.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581444+0000", 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581444+0000", 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581444+0000", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4700000000000002e-07 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581446+0000", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0279999999999999e-06 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581452+0000", 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2319999999999998e-06 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581458+0000", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1139999999999997e-06 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583352+0000", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018944369999999999 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583358+0000", 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2829999999999998e-06 2024-09-13T14:35:11.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762814+0000", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1794560810000001 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.762946+0000", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013214800000000001 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.763212+0000", 2024-09-13T14:35:11.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026515300000000001 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764829+0000", 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016173629999999999 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.767194+0000", 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023654679999999999 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769679+0000", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024843909999999999 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770015+0000", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00033647000000000001 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770174+0000", 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015856999999999999 2024-09-13T14:35:11.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770220+0000", 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5846000000000001e-05 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770295+0000", 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5233000000000003e-05 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214395 6.18 6:1c19bb6d:::benchmark_data_smithi017_185083_object89783:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581465+0000", 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225443018999997, 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.189105012, 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214395 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.906 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581465+0000", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581465+0000", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581466+0000", 2024-09-13T14:35:11.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7700000000000005e-07 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581468+0000", 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6290000000000002e-06 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581475+0000", 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3219999999999998e-06 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581481+0000", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6279999999999999e-06 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583383+0000", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001901483 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583390+0000", 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7909999999999999e-06 2024-09-13T14:35:11.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.764927+0000", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.181536946 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.765050+0000", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012345800000000001 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.765335+0000", 2024-09-13T14:35:11.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028524900000000001 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769697+0000", 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043613300000000001 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769738+0000", 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1202999999999998e-05 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769928+0000", 2024-09-13T14:35:11.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019037899999999999 2024-09-13T14:35:11.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770127+0000", 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019843500000000001 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770396+0000", 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000269411 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770443+0000", 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6842000000000003e-05 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770570+0000", 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127429 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.914 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214495 6.18 6:1d956855:::benchmark_data_smithi017_185083_object89883:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581489+0000", 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225419341000006, 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1893286789999999, 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.915 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214495 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581489+0000", 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581489+0000", 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581489+0000", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8099999999999998e-07 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581491+0000", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0020000000000001e-06 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581498+0000", 2024-09-13T14:35:11.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.224e-06 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581503+0000", 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2259999999999996e-06 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583413+0000", 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001909963 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583419+0000", 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6460000000000004e-06 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.767226+0000", 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183806779 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.767351+0000", 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012443799999999999 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.767648+0000", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029723500000000002 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.769832+0000", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021844049999999999 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770359+0000", 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052651399999999995 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770615+0000", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025633399999999999 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770657+0000", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1523000000000002e-05 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770693+0000", 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6541000000000001e-05 2024-09-13T14:35:11.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770731+0000", 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7956e-05 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770818+0000", 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6512000000000001e-05 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214512 6.18 6:188ef604:::benchmark_data_smithi017_185083_object89900:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581509+0000", 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225399592000002, 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.198636128, 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214512 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581509+0000", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581509+0000", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581509+0000", 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4799999999999999e-07 2024-09-13T14:35:11.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581511+0000", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.624e-06 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581517+0000", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581522+0000", 2024-09-13T14:35:11.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9359999999999998e-06 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583444+0000", 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019222219999999999 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583450+0000", 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7399999999999998e-06 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770851+0000", 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.18740059 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.770940+0000", 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999994e-05 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.771212+0000", 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027241500000000003 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.775230+0000", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040180679999999996 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.775278+0000", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7076999999999997e-05 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779701+0000", 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004423771 2024-09-13T14:35:11.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779791+0000", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9341999999999999e-05 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779940+0000", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014894800000000001 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780007+0000", 2024-09-13T14:35:11.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7459000000000006e-05 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780145+0000", 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137897 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214586 6.18 6:1813a1d9:::benchmark_data_smithi017_185083_object89974:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581527+0000", 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225381122000002, 2024-09-13T14:35:11.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.202780513, 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214586 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581527+0000", 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581527+0000", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581528+0000", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.03e-07 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581534+0000", 2024-09-13T14:35:11.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.917e-06 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581541+0000", 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2419999999999997e-06 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581547+0000", 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5810000000000002e-06 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583777+0000", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022301399999999998 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583786+0000", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1049999999999995e-06 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.773179+0000", 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1893932 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.773298+0000", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119647 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.773575+0000", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027680800000000002 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779675+0000", 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060996920000000003 2024-09-13T14:35:11.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779812+0000", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013735900000000001 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779869+0000", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6529e-05 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780237+0000", 2024-09-13T14:35:11.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036771299999999998 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784116+0000", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038791670000000002 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784183+0000", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7664999999999997e-05 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784308+0000", 2024-09-13T14:35:11.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012444499999999999 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214643 6.18 6:18927a30:::benchmark_data_smithi017_185083_object90031:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581554+0000", 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225354795000001, 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2031559220000001, 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.938 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214643 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581554+0000", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581554+0000", 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581554+0000", 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3700000000000001e-07 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581556+0000", 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0640000000000001e-06 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:11.941 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:11.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581563+0000", 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9920000000000001e-06 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581569+0000", 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4620000000000004e-06 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583817+0000", 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002248271 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583824+0000", 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7490000000000001e-06 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.775309+0000", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.191485516 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.775421+0000", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011147200000000001 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.775740+0000", 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031986500000000001 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.779743+0000", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040029339999999997 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780162+0000", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000418949 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.783990+0000", 2024-09-13T14:35:11.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038279870000000001 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784063+0000", 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2821e-05 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784367+0000", 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030406299999999998 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784578+0000", 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0002107 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784710+0000", 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013164000000000001 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214652 6.18 6:1faeac48:::benchmark_data_smithi017_185083_object90040:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581577+0000", 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225331131000004, 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2323358360000001, 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214652 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581577+0000", 2024-09-13T14:35:11.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581577+0000", 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581578+0000", 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8099999999999998e-07 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581580+0000", 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2579999999999998e-06 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581586+0000", 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.477e-06 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581592+0000", 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.135e-06 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583913+0000", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023216080000000002 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583920+0000", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.007e-06 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.777470+0000", 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.193550133 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.777601+0000", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130801 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.777882+0000", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028083600000000002 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784009+0000", 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061269080000000004 2024-09-13T14:35:11.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784779+0000", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077063299999999995 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784837+0000", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7590000000000003e-05 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.793696+0000", 2024-09-13T14:35:11.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008859423 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.813688+0000", 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019991871000000001 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.813753+0000", 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4491000000000003e-05 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.813913+0000", 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016028400000000001 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214673 6.18 6:1af9450b:::benchmark_data_smithi017_185083_object90061:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581599+0000", 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225309881000001, 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2325423870000001, 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214673 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581599+0000", 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581599+0000", 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581599+0000", 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9700000000000002e-07 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581601+0000", 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.823e-06 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581607+0000", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8100000000000003e-06 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581612+0000", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3619999999999997e-06 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583949+0000", 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023368159999999998 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.583956+0000", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0790000000000003e-06 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780278+0000", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1963219380000001 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780362+0000", 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3772999999999999e-05 2024-09-13T14:35:11.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.780626+0000", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026415500000000002 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784015+0000", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003389342 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784895+0000", 2024-09-13T14:35:11.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088020100000000005 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.793458+0000", 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085626710000000009 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.793734+0000", 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027629299999999999 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.813963+0000", 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020229319999999999 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.814017+0000", 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4007e-05 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.814141+0000", 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012339999999999999 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.962 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214675 6.18 6:1cd6bf2a:::benchmark_data_smithi017_185083_object90063:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e231)", 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:01.581619+0000", 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.225289734, 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2328086119999999, 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214675 2024-09-13T14:35:11.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581619+0000", 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581619+0000", 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:35:11.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581619+0000", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8000000000000001e-07 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581622+0000", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4820000000000001e-06 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581631+0000", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3389999999999999e-06 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:35:11.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.581636+0000", 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3140000000000003e-06 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.584547+0000", 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029109800000000001 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:01.584556+0000", 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1779999999999992e-06 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.782088+0000", 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1975313329999999 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.782216+0000", 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012811800000000001 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1,3,7", 2024-09-13T14:35:11.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.782518+0000", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030194400000000001 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.784722+0000", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022042540000000001 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.788376+0000", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036544419999999999 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.793524+0000", 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051473600000000001 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.793765+0000", 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000240988 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.814180+0000", 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020414907999999999 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.814302+0000", 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122299 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:02.814427+0000", 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012529300000000001 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:35:11.971 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:35:12.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:12.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:12.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:12.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:12.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:12.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:12.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:12.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:12.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6217 6201 193.766 172.375 0.00689776 0.00515173 2024-09-13T14:35:12.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:12.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:12.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:12.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:12.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:12.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[103001]: pgmap v682: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[107556]: pgmap v682: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:12.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:35:12.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:12.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:12.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:12.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:12.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:12 smithi189 ceph-mon[90992]: pgmap v682: 99 pgs: 99 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:35:13.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:13.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:13.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:13.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:13.260 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:13.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:13.321 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:35:13.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:13.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:13.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:13.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9276 9260 192.901 191.188 0.0121257 0.00517339 2024-09-13T14:35:13.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:13.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:13.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:35:13.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:13.691 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:13.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:13.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:13.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:13.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:13.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:13.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:14.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:14.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:14.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:14.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:14.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:14.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:14.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:14.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:14.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:14.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12057 12041 188.125 173.812 0.00707641 0.00530744 2024-09-13T14:35:14.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:14.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:14.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:35:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[103001]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:35:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[103001]: pgmap v684: 99 pgs: 99 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:35:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:35:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[107556]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:35:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:14 smithi017 ceph-mon[107556]: pgmap v684: 99 pgs: 99 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:35:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/471885424' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:35:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:14 smithi189 ceph-mon[90992]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:35:14.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:14 smithi189 ceph-mon[90992]: pgmap v684: 99 pgs: 99 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2024-09-13T14:35:14.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:14.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:14.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:14.934 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:14.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:15.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:15.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:15.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:15.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:15.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:15.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:15.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:15.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:15.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:15.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 14652 14636 182.936 162.188 0.00508638 0.00545899 2024-09-13T14:35:15.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:15.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:15.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:15.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:15.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:15.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:15.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:35:15.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:15.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:15.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:16.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:16.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:16.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:16.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:16.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:16.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:16.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:16.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:16.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 17143 17127 178.382 155.688 0.00531295 0.00559623 2024-09-13T14:35:16.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:16.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:16 smithi017 ceph-mon[107556]: pgmap v685: 99 pgs: 99 active+clean; 510 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:35:16.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:16 smithi017 ceph-mon[103001]: pgmap v685: 99 pgs: 99 active+clean; 510 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:35:16.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:16.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:16.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:16 smithi189 ceph-mon[90992]: pgmap v685: 99 pgs: 99 active+clean; 510 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:35:16.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:16.859 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:16.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:16.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:17.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:17.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:17.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:17.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:17.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:17.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:17.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:17.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:17.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:17.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:17.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 19770 19754 176.352 164.188 0.00546487 0.00566367 2024-09-13T14:35:17.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:17.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:17] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-09-13T14:35:17.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:17.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:17.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:17.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:17.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:18.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:18.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:18.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:18.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:18.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:18.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:18.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:18.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:18.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:18.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 22147 22131 172.877 148.562 0.0071111 0.00577905 2024-09-13T14:35:18.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:18.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:18.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:18.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:18 smithi189 ceph-mon[90992]: pgmap v686: 99 pgs: 99 active+clean; 617 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:35:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:18.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:18.895 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:18.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:18.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:18 smithi017 ceph-mon[103001]: pgmap v686: 99 pgs: 99 active+clean; 617 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:35:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:18 smithi017 ceph-mon[107556]: pgmap v686: 99 pgs: 99 active+clean; 617 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:35:18.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:19.027 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:19.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:19.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:19.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:19.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:19.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:19.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:19.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:19.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 24384 24368 169.202 139.812 0.0128213 0.00590426 2024-09-13T14:35:19.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:19.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:19.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:19.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:19.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:19.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:19.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:20.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:20.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:20.119 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:20.153 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:20.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:20.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:20.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:20.416 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:20.429 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:20 smithi189 ceph-mon[90992]: pgmap v687: 99 pgs: 99 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:35:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:20.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:20.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:20.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 26444 26428 165.152 128.75 0.00910299 0.00604854 2024-09-13T14:35:20.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:20.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:20 smithi017 ceph-mon[103001]: pgmap v687: 99 pgs: 99 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:35:20.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:20.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:20 smithi017 ceph-mon[107556]: pgmap v687: 99 pgs: 99 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:35:20.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:20.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:20.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:20.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:20.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:20.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:20.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:21.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:21.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:21.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:21.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:21.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:21.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:21.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:21.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:21.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:21.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 28126 28110 159.694 105.125 0.00390689 0.00625554 2024-09-13T14:35:21.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:21.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:21.725 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:21.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:21.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:21.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:21.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:22.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:22.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:22.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:22.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:22.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:22.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:22.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:22.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:22.481 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:35:22.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:22.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 30069 30053 156.506 121.438 0.00501197 0.00638166 2024-09-13T14:35:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:22.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:22.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[103001]: pgmap v688: 99 pgs: 99 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:35:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3328411024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1187440594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[107556]: pgmap v688: 99 pgs: 99 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:35:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3328411024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1187440594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:22.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:22.745 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:22 smithi189 ceph-mon[90992]: pgmap v688: 99 pgs: 99 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:35:22.745 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3328411024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.745 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:35:22.745 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1187440594' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:22.821 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:35:22.839 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:35:22.840 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:35:22.841 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:35:22.841 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:35:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:22.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:22.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:22.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:22.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:23.064 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:23.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:23.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:23.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:23.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:23.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:23.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:23.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:23.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:23.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:23.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 32171 32155 154.571 131.375 0.00710483 0.00646442 2024-09-13T14:35:23.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:23.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:23 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:35:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:23 smithi189 ceph-mon[90992]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:35:23.756 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:23.757 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:35:23.757 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[103001]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:35:23.757 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:23.757 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:35:23.757 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:23 smithi017 ceph-mon[107556]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:35:23.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:23.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:23.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:23.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:23.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:23.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:24.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:24.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:24.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:24.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:24.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:24.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:24.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:24.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:24.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:24.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 33932 33916 151.392 110.062 0.00553657 0.00659808 2024-09-13T14:35:24.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:24.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:24 smithi017 ceph-mon[103001]: pgmap v690: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:35:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:24 smithi017 ceph-mon[107556]: pgmap v690: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:35:24.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:24.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:24 smithi189 ceph-mon[90992]: pgmap v690: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:35:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:24.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:24.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:24.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:24.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:25.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:25.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:25.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:25.261 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:25.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:25.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:25.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:25.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:25.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 35488 35472 147.782 97.25 0.0111131 0.00675984 2024-09-13T14:35:25.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:25.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:25.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:25.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:35:25.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:25.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:25.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:25.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:25.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:25.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:26.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:26.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:26.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:26.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:26.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:26.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:26.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:26.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:26.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 37110 37094 144.88 101.375 0.00842343 0.00689629 2024-09-13T14:35:26.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:26.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:26 smithi189 ceph-mon[90992]: pgmap v691: 99 pgs: 99 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T14:35:26.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:26.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:26.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:26 smithi017 ceph-mon[103001]: pgmap v691: 99 pgs: 99 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T14:35:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:26 smithi017 ceph-mon[107556]: pgmap v691: 99 pgs: 99 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T14:35:26.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:26.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:26.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:27.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:27.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:27.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:27.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:27.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:27.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:27.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:27.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:27.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 38937 38921 143.043 114.188 0.0141583 0.00698408 2024-09-13T14:35:27.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:27.600 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:35:27.600 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:35:27.600 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:27] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-09-13T14:35:27.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:27.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:35:27.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:27.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:27.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:27.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:27.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:27.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[103001]: pgmap v692: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:35:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:27 smithi017 ceph-mon[107556]: pgmap v692: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:35:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:28.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:28.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:28.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:27 smithi189 ceph-mon[90992]: pgmap v692: 99 pgs: 99 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:35:28.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:28.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:28.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:28.164 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:28.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:28.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:28.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:28.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:28.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:28.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:28.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:28.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 40528 40512 140.62 99.4375 0.00801612 0.00710541 2024-09-13T14:35:28.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:28.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:28.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:35:28.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:28.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:28.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:28.885 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:28.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:28.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:29.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:29.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:29.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:29.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:29.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:29.357 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:35:29.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:29.382 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1288958010' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.382 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1288958010' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:29.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:29.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 15 41910 41895 137.769 86.4375 0.00781827 0.00725309 2024-09-13T14:35:29.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:29.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1288958010' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:35:29.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:29.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:29.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:29.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:29.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:30.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:30.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:30.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:30.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:30.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:30.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:30.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:35:30.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:35:30.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[103001]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:35:30.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[103001]: pgmap v694: 99 pgs: 99 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:35:30.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:35:30.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[107556]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:35:30.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:30 smithi017 ceph-mon[107556]: pgmap v694: 99 pgs: 99 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:35:30.445 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:30.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:30 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:35:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:30 smithi189 ceph-mon[90992]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:35:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:30 smithi189 ceph-mon[90992]: pgmap v694: 99 pgs: 99 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:35:30.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:35:30.562167+0000 min lat: 0.00129139 max lat: 0.128255 avg lat: 0.00732393 2024-09-13T14:35:30.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:30.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 43691 43675 136.443 111.25 0.00730889 0.00732393 2024-09-13T14:35:30.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:30.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:30.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:30.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:30.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:30.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:30.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:30.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:30.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:31.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:31.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:31.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:31.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:31.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:31.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:31.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:31.425 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:31.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:31.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:31.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 44966 44950 133.74 79.6875 0.006699 0.00747151 2024-09-13T14:35:31.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:31.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:31.762 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:31.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:31.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:31.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:31.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:31.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:31.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:32.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:32.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:32.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:32.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:32.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:32.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:32.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:32.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:32.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:32.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 46336 46320 131.553 85.625 0.0115663 0.00759632 2024-09-13T14:35:32.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:32.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:32 smithi189 ceph-mon[90992]: pgmap v695: 99 pgs: 99 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-09-13T14:35:32.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:32.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:32 smithi017 ceph-mon[103001]: pgmap v695: 99 pgs: 99 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-09-13T14:35:32.815 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:32 smithi017 ceph-mon[107556]: pgmap v695: 99 pgs: 99 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-09-13T14:35:32.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:32.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:32.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:32.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:33.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:33.068 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:33.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:33.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:33.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:33.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:33.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:33.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:33.468 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:33.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 47917 47901 130.13 98.8125 0.0062366 0.00767939 2024-09-13T14:35:33.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:33.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:33.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:33.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:33.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:33.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:33.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:34.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:34.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:34.104 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:34.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:34.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:34.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:34.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:34.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:34.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:34.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 49436 49420 128.663 94.9375 0.00848022 0.00776636 2024-09-13T14:35:34.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:34.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:34.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:34 smithi017 ceph-mon[103001]: pgmap v696: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:35:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:34 smithi017 ceph-mon[107556]: pgmap v696: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:35:34.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:34.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:34 smithi189 ceph-mon[90992]: pgmap v696: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:35:34.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:34.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:34.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:34.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:35.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:35.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:35.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:35.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:35.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:35.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:35.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:35.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:35.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:35.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:35.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 50999 50983 127.424 97.6875 0.00711718 0.00784198 2024-09-13T14:35:35.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:35.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:35.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:35.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:35.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:35.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:35.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:35.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:36.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:36.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:36.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:36.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:36.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:36.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:36.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:36.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:36.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:36 smithi017 ceph-mon[103001]: pgmap v697: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:35:36.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:36 smithi017 ceph-mon[107556]: pgmap v697: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:35:36.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 52547 52531 126.244 96.75 0.00844937 0.00791525 2024-09-13T14:35:36.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:36.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:36.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:36 smithi189 ceph-mon[90992]: pgmap v697: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-13T14:35:36.761 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:36.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:36.891 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:36.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:36.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:36.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:37.022 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:37.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:37.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:37.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:37.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:37.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:37.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:37.446 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:37.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:37.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 53969 53953 124.86 88.875 0.0129837 0.00800153 2024-09-13T14:35:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:37.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:35:37.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:35:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:35:37.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1492969078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:37.608 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:37] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-09-13T14:35:37.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1492969078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:37.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:37.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1492969078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:37.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:37.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:37.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:37.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:37.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:38.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:38.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:38.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:38.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:38.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:38.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:38.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:38.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:38.563 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 55369 55353 123.525 87.5 0.0137209 0.0080898 2024-09-13T14:35:38.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:38.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:38 smithi189 ceph-mon[90992]: pgmap v698: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-13T14:35:38.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:38.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:38.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:38.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:38.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:38 smithi017 ceph-mon[103001]: pgmap v698: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-13T14:35:38.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:38 smithi017 ceph-mon[107556]: pgmap v698: 99 pgs: 99 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-13T14:35:38.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:38.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:39.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:39.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:39.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:39.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:39.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:39.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:39.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:39.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:39.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:39.473 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:39.508 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:35:39.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:39.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:39.563 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 56815 56799 122.382 90.375 0.0298801 0.00816444 2024-09-13T14:35:39.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:39.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:39.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:39.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:35:39.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:39.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:39.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:39.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:40.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:40.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:40.145 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:40.247 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:40.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:40.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:40.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:40.493 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:40.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:40.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:40.563 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 58193 58177 121.173 86.125 0.0157892 0.00824762 2024-09-13T14:35:40.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:40.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:40 smithi189 ceph-mon[90992]: pgmap v699: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:35:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:40 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:35:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:40 smithi189 ceph-mon[90992]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:35:40.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:40.796 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:40.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:40.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:40.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[103001]: pgmap v699: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:35:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:35:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[103001]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:35:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[107556]: pgmap v699: 99 pgs: 99 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:35:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:35:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3763655521' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:35:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:40 smithi017 ceph-mon[107556]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:35:41.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:41.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:41.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:41.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:41.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:41.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:41.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:41.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:41.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:41.563 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 59533 59517 119.966 83.75 0.00659677 0.00833024 2024-09-13T14:35:41.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:41.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:41.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:41.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:41.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:41.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:41.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:41.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:35:42.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:42.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:42.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:42.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:42.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:42.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:42.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:42.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:42.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:42.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 60840 60824 118.765 81.6875 0.0137647 0.00841385 2024-09-13T14:35:42.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:42.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:42.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:42 smithi189 ceph-mon[90992]: pgmap v701: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:35:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[103001]: pgmap v701: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[107556]: pgmap v701: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:42.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:42.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:42.857 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:42.959 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:42.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:42.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:43.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:43.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:43.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:43.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:43.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:43.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:43.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:43.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:43.521 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:43.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 62226 62210 117.791 86.625 0.00834824 0.00848457 2024-09-13T14:35:43.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:43.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:43.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:43.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:43.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:43.912 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:43.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:43.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:43.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:44.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:44.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:44.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:44.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:44.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:44.344 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:44.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:44.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:44.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 63574 63558 116.804 84.25 0.00922587 0.00855468 2024-09-13T14:35:44.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:44.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:44.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:44.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:44.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:44.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:35:44.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:44 smithi017 ceph-mon[103001]: pgmap v702: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:35:44.847 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:44 smithi017 ceph-mon[107556]: pgmap v702: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:35:44.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:44.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:44.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:44.991 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:35:44.991 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:35:44.991 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:35:44.991 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:35:44.991 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:35:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:44 smithi189 ceph-mon[90992]: pgmap v702: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:35:45.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:45.049 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:45.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:45.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:45.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:45.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:45.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:45.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:45.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:45.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 64830 64814 115.71 78.5 0.0120572 0.00863693 2024-09-13T14:35:45.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:45.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:45.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:45.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:45.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:35:45.825 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:45.851 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:45.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:45.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:45.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:45.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:46.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:46.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:46.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:46.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:46.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:46.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:46.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:46.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:46.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:46.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 66101 66085 114.702 79.4375 0.00915418 0.00871256 2024-09-13T14:35:46.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:46.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:46 smithi017 ceph-mon[103001]: pgmap v703: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:35:46.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:46 smithi017 ceph-mon[107556]: pgmap v703: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:35:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:46.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:46.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:46.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:46.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:46.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:46.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:46 smithi189 ceph-mon[90992]: pgmap v703: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:35:47.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:47.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:47.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:47.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:47.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:47.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:47.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:47.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:47.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 67269 67253 113.575 73 0.00667832 0.0087957 2024-09-13T14:35:47.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:47.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:47.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:47] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:35:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:35:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:35:47.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:47.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:47.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:47.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:47.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:35:48.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:48.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:48.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:48.192 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:48.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:48.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:48.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:48.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:48.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:48.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 68623 68607 112.812 84.625 0.0162838 0.0088586 2024-09-13T14:35:48.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:48.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:48.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:48.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:48.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:48.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:48.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:48.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:48.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[103001]: pgmap v704: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:35:48.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:35:48.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[103001]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:35:48.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[107556]: pgmap v704: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:35:48.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:35:48.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:48 smithi017 ceph-mon[107556]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:35:48.928 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:35:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:48 smithi189 ceph-mon[90992]: pgmap v704: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:35:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2660063474' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:35:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:48 smithi189 ceph-mon[90992]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:35:49.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:49.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:49.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:49.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:49.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:49.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:49.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:49.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:49.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 69908 69892 111.978 80.3125 0.0228153 0.00892376 2024-09-13T14:35:49.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:49.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:49.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:49.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:49.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:49.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:49.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:49.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:50.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:50.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:50.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:50.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:50.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:50.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:50.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:50.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:50.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:35:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:35:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:50.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:50.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:35:50.565895+0000 min lat: 0.00129139 max lat: 0.128255 avg lat: 0.00900089 2024-09-13T14:35:50.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:50.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 71098 71082 111.038 74.375 0.0128551 0.00900089 2024-09-13T14:35:50.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:50.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:50.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:50.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:50.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:50.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:50 smithi017 ceph-mon[103001]: pgmap v706: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:35:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:50 smithi017 ceph-mon[107556]: pgmap v706: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:35:50.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:50.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:50.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:50 smithi189 ceph-mon[90992]: pgmap v706: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:35:51.089 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:51.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:51.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:51.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:51.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:35:51.330 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:35:51.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:51.367 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:35:51.368 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:35:51.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:51.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:35:51.410 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:35:51.450 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:35:51.451 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:35:51.481 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:35:51.482 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:35:51.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:51.515 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:35:51.515 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:35:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:51.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:51.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:35:51.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:35:51.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 72271 72255 110.118 73.3125 0.0160219 0.00907566 2024-09-13T14:35:51.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:35:51.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:35:51.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:51.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3045621376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3045621376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:51.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:51.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:51.924 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:51.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:51.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:51.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3045621376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:35:52.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:52.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:52.170 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:52.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:52.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:35:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:52.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:52.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:52.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:52.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:52.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:52.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:52.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 73411 73395 109.191 71.25 0.0138642 0.00915337 2024-09-13T14:35:52.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:52.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:52.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:52.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:52.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:52.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:52 smithi017 ceph-mon[103001]: pgmap v707: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:35:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:52 smithi017 ceph-mon[107556]: pgmap v707: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:35:52.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:52.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:52.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:52 smithi189 ceph-mon[90992]: pgmap v707: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:35:53.056 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:53.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:53.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:53.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:53.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:53.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:53.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:53.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:53.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:53.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:53.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 74582 74566 108.354 73.1875 0.0191496 0.00922205 2024-09-13T14:35:53.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:53.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:53.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:53.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:53.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:53.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:53.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:54.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:54.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:54.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:54.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:54.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:54.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:54.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:54.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:54.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 75623 75607 107.37 65.0625 0.0101446 0.00930763 2024-09-13T14:35:54.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:54.651 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:54.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:54.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:54.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:54.904 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:54.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:54 smithi017 ceph-mon[103001]: pgmap v708: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:35:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:54 smithi017 ceph-mon[107556]: pgmap v708: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:35:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:54 smithi189 ceph-mon[90992]: pgmap v708: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:35:55.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:55.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:55.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:35:55.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:55.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:55.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:55.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:55.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:55.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:55.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 76611 76595 106.356 61.75 0.0112366 0.00939674 2024-09-13T14:35:55.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:55.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:55.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:55.831 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:55.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:55.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:55.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:56.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:56.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:56.205 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:56.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:56.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:56.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:56.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:56.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:56.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 77671 77655 105.484 66.25 0.0105068 0.00947372 2024-09-13T14:35:56.624 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:56.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:56.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:56.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[103001]: pgmap v709: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[107556]: pgmap v709: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:56.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:56 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:56.874 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:56.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:56.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:56.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:56.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:56 smithi189 ceph-mon[90992]: pgmap v709: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-13T14:35:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:56 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:57.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:35:57.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:35:57.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:57.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:57.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:57.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:57.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:57.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:57.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:35:57] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-09-13T14:35:57.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:57.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:57.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 78676 78660 104.576 62.8125 0.0105295 0.00955755 2024-09-13T14:35:57.603 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:35:57.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:57.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:57.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:57.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:57.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/470437351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/470437351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:57.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:57.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:57.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:35:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/470437351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:35:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:35:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:58.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:35:58.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:58.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:35:58.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:58.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:58.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:58.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:58.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:58.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 79874 79858 103.957 74.875 0.0131011 0.0096139 2024-09-13T14:35:58.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:58.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:58.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:58.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:58.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: pgmap v710: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:35:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T14:35:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:35:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[103001]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: pgmap v710: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:35:58.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:58 smithi017 ceph-mon[107556]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:35:58.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:58.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: pgmap v710: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:35:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-13T14:35:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:35:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:35:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:35:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:58 smithi189 ceph-mon[90992]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:35:59.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:35:59.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:59.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:35:59.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:35:59.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:35:59.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:59.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:59.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:35:59.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:35:59.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 81035 81019 103.317 72.5625 0.00684983 0.00967393 2024-09-13T14:35:59.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:35:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:59.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:35:59.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:35:59.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:35:59.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:35:59.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:35:59 smithi017 ceph-mon[103001]: pgmap v712: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:35:59.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:35:59 smithi017 ceph-mon[107556]: pgmap v712: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:35:59.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:35:59.955 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:35:59.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:35:59.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:35:59 smithi189 ceph-mon[90992]: pgmap v712: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:36:00.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:00.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:00.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:00.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:00.263 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:00.263 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:00.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:00.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:00.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:00.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:00.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:00.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 82164 82148 102.661 70.5625 0.0082694 0.00973488 2024-09-13T14:36:00.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:00.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:00.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:00.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:00.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:00.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:00.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:01.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:01.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:01.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:36:01.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:01.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:01.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:01.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:01.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:01.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:01.419 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:01.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:01.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:01.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 83309 83293 102.052 71.5625 0.00618205 0.00979406 2024-09-13T14:36:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:01.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:01.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:01.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:01.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:01.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:01.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:01.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:01.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:02.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:02.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:02.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:02 smithi017 ceph-mon[103001]: pgmap v713: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:36:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:02 smithi017 ceph-mon[107556]: pgmap v713: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:36:02.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:02.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:02 smithi189 ceph-mon[90992]: pgmap v713: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:36:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:02.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:02.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:02.402 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:02.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:02.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:02.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:02.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 84382 84366 101.379 67.0625 0.0211421 0.00985857 2024-09-13T14:36:02.631 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:02.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:02.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:02.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:02.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:02.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:02.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:03.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:03.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:03.094 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:36:03.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:03.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:03.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:03.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:03.354 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:03.354 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:03.355 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/440695903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.355 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:03 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:03.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/440695903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:03 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/440695903' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:03 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:36:03.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:03.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:03.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 85408 85392 100.676 64.125 0.0151175 0.0099271 2024-09-13T14:36:03.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:03.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:03.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:03.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:03.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:03.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:03.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:03.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:03.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:36:04.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:04.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:04.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:04.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:04.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:04.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:04.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:04.342 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:04.445 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:04.446 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:04 smithi189 ceph-mon[90992]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:36:04.446 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:04 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:36:04.446 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:04 smithi189 ceph-mon[90992]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:36:04.446 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3077803615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[103001]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[103001]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3077803615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[107556]: pgmap v714: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[107556]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:36:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3077803615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:04.467 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:04.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:36:04.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 86308 86292 99.8531 56.25 0.0146307 0.0100099 2024-09-13T14:36:04.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:04.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:04.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:04.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:04.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:04.871 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:04.892 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:04.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:04.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:05.136 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:36:05.137 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:36:05.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:05.166 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:05.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:05.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:05.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:05.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:05.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 87370 87354 99.2444 66.375 0.00821583 0.0100698 2024-09-13T14:36:05.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:05.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:05.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:05.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:05.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:05.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:05.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:05.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:06.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:36:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:06.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:06.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:06.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:06.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:06.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:06.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:06.388 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:06 smithi189 ceph-mon[90992]: pgmap v716: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:36:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:06 smithi017 ceph-mon[107556]: pgmap v716: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:36:06.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:06.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:06 smithi017 ceph-mon[103001]: pgmap v716: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:36:06.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:06.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 88421 88405 98.6452 65.6875 0.00520596 0.0101318 2024-09-13T14:36:06.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:06.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:06.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:06.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:06.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:06.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:06.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:06.994 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:07.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:07.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:07.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:07.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:07.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:07.258 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:07.258 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:36:07.258 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:36:07.258 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:36:07.258 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:07.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:07.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:07.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:07.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:07] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-09-13T14:36:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:07.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:07.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:07.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 89440 89424 98.0318 63.6875 0.00674483 0.0101957 2024-09-13T14:36:07.578 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:07.680 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:07.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:07.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:07.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:07.916 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:07.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:07.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:07.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:08.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:36:08.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:08.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:08.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:08.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:08.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:08.410 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:08.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:08 smithi017 ceph-mon[103001]: pgmap v717: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:36:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:08 smithi017 ceph-mon[107556]: pgmap v717: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:36:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:08 smithi189 ceph-mon[90992]: pgmap v717: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:36:08.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:08.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:08.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 15 90529 90514 97.5162 68.125 0.0197962 0.0102494 2024-09-13T14:36:08.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:08.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:08.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:08.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:08.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:08.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:36:08.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:08.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:08.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:09.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:09.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:09.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:09.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:09.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:09.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:09.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:09.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:09.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 91530 91514 96.9226 62.5 0.0162544 0.0103119 2024-09-13T14:36:09.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:09.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:09.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:09.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:09.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:09.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:09.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:10.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:10.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:10.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:10.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:10.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:10.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:10.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:10 smithi017 ceph-mon[103001]: pgmap v718: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:36:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:10 smithi017 ceph-mon[107556]: pgmap v718: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:36:10.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:10.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:10 smithi189 ceph-mon[90992]: pgmap v718: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:36:10.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:36:10.568429+0000 min lat: 0.00129139 max lat: 0.128255 avg lat: 0.0103803 2024-09-13T14:36:10.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:36:10.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 92446 92430 96.2614 57.25 0.00878889 0.0103803 2024-09-13T14:36:10.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:10.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:10.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:10.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:10.862 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:10.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:10.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:10.939 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:11.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:11.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:11.141 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:36:11.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1526358018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.158 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1526358018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.158 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:11.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241019 5.19 5:9bc7a0be:::benchmark_data_smithi017_147622_object106390:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 409.778840742, 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.256814476, 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.270 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241019 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486707+0000", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486706+0000", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486709+0000", 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9229999999999998e-06 2024-09-13T14:36:11.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486716+0000", 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0690000000000004e-06 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486720+0000", 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9430000000000002e-06 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486807+0000", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6104000000000001e-05 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486811+0000", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9119999999999997e-06 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491474+0000", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046621689999999999 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491479+0000", 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5219999999999998e-06 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736782+0000", 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.245302446 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.736872+0000", 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0799999999999998e-05 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.737235+0000", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036251399999999998 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743274+0000", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0060387319999999998 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743285+0000", 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1610000000000001e-05 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743340+0000", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5067e-05 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743376+0000", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6161999999999997e-05 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743417+0000", 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0441000000000001e-05 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743451+0000", 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3683999999999998e-05 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743521+0000", 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0822999999999996e-05 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.278 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241060 5.19 5:9f1c8886:::benchmark_data_smithi017_147622_object106431:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 409.77882280199998, 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.265054519, 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:11.279 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241060 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486725+0000", 2024-09-13T14:36:11.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486724+0000", 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486727+0000", 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.39e-06 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486745+0000", 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8088e-05 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486751+0000", 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5740000000000003e-06 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486922+0000", 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017172699999999999 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486928+0000", 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491499+0000", 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045716660000000003 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491504+0000", 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7899999999999999e-06 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.740366+0000", 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2488624189999999 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.740483+0000", 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116347 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.741017+0000", 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053390399999999998 2024-09-13T14:36:11.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743385+0000", 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023684090000000001 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743558+0000", 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000173222 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751434+0000", 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078755990000000005 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751517+0000", 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3270999999999996e-05 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751580+0000", 2024-09-13T14:36:11.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2535999999999999e-05 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751671+0000", 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1272000000000003e-05 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751779+0000", 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010842500000000001 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241063 5.19 5:9e8b5f5a:::benchmark_data_smithi017_147622_object106434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 409.77870693800003, 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.272999937, 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:11.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241063 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486841+0000", 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486840+0000", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486849+0000", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2599999999999994e-06 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486860+0000", 2024-09-13T14:36:11.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0482999999999999e-05 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486865+0000", 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.361e-06 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486954+0000", 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9098999999999995e-05 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486959+0000", 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0239999999999997e-06 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491524+0000", 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045650200000000004 2024-09-13T14:36:11.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491529+0000", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8980000000000004e-06 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743595+0000", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2520661099999999 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.743721+0000", 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000125845 2024-09-13T14:36:11.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.744209+0000", 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00048819400000000001 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751400+0000", 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071904550000000001 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751834+0000", 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00043387900000000002 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751878+0000", 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4051999999999998e-05 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751911+0000", 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3725e-05 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759666+0000", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077542669999999996 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759719+0000", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3158000000000002e-05 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759841+0000", 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012194 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.298 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:241075 5.19 5:994d542c:::benchmark_data_smithi017_147622_object106446:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 409.77867727900002, 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2733477289999999, 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:11.299 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 241075 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486870+0000", 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486873+0000", 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3620000000000002e-06 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486881+0000", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8469999999999996e-06 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486885+0000", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2039999999999999e-06 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486974+0000", 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9004000000000004e-05 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.486977+0000", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2850000000000001e-06 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491625+0000", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046485279999999999 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:21.491631+0000", 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7810000000000002e-06 2024-09-13T14:36:11.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747092+0000", 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2554612540000001 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747215+0000", 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012232599999999999 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,5,6,7", 2024-09-13T14:36:11.304 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.747694+0000", 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00047920500000000002 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.751946+0000", 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042521900000000003 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759556+0000", 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076104850000000002 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759602+0000", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5800999999999997e-05 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.759899+0000", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00029672100000000001 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760044+0000", 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000145549 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760096+0000", 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1566000000000001e-05 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:29:22.760218+0000", 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012210199999999999 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.307 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213567 6.17 6:ebec8fbe:::benchmark_data_smithi017_185083_object88955:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.577016+0000", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.688531423, 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1028458909999999, 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213567 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.308 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577016+0000", 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577016+0000", 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577016+0000", 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577020+0000", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5399999999999997e-06 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577030+0000", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5279999999999992e-06 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577035+0000", 2024-09-13T14:36:11.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4419999999999997e-06 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.583297+0000", 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006261661 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op must wait for map", 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.583303+0000", 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2349999999999996e-06 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.584415+0000", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001112198 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.584424+0000", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9190000000000005e-06 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571688+0000", 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98726444899999999 2024-09-13T14:36:11.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571699+0000", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0631999999999999e-05 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.679769+0000", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.108069559 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.679862+0000", 2024-09-13T14:36:11.313 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3482999999999999e-05 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213575 6.17 6:ef706e1c:::benchmark_data_smithi017_185083_object88963:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.577044+0000", 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.68850379899999, 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1029053000000002, 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.314 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213575 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577044+0000", 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577044+0000", 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577043+0000", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577046+0000", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6910000000000002e-06 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577052+0000", 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1369999999999999e-06 2024-09-13T14:36:11.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.577056+0000", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2220000000000004e-06 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.588963+0000", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011906236000000001 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.588971+0000", 2024-09-13T14:36:11.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7690000000000002e-06 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571737+0000", 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98276591599999996 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571746+0000", 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4999999999999999e-06 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.318 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.679887+0000", 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.10814134 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.679949+0000", 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1997999999999998e-05 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213597 6.17 6:ecee8d97:::benchmark_data_smithi017_185083_object88985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.590264+0000", 2024-09-13T14:36:11.319 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.67528407099999, 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1151528329999998, 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213597 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590264+0000", 2024-09-13T14:36:11.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590264+0000", 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590262+0000", 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590268+0000", 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4070000000000001e-06 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590278+0000", 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8539999999999992e-06 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.590283+0000", 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1830000000000002e-06 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.322 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.592375+0000", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0020918970000000001 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.592382+0000", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8469999999999996e-06 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571791+0000", 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97940823200000005 2024-09-13T14:36:11.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571799+0000", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5129999999999996e-06 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.679981+0000", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.108181928 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.680112+0000", 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000130479 2024-09-13T14:36:11.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.680485+0000", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00037352499999999999 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.689018+0000", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085331790000000001 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705166+0000", 2024-09-13T14:36:11.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016147814 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705205+0000", 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8778e-05 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705284+0000", 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9467000000000003e-05 2024-09-13T14:36:11.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705332+0000", 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7583999999999997e-05 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705416+0000", 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4506999999999996e-05 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213731 6.17 6:efa746c2:::benchmark_data_smithi017_185083_object89119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.327 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.697354+0000", 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.568193347, 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0081755590000001, 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213731 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:11.330 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697354+0000", 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697354+0000", 2024-09-13T14:36:11.331 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697353+0000", 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.332 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697361+0000", 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3990000000000004e-06 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697375+0000", 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4328000000000001e-05 2024-09-13T14:36:11.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697382+0000", 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1400000000000002e-06 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697446+0000", 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3220999999999997e-05 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.697455+0000", 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2420000000000006e-06 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571838+0000", 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87438343900000004 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571843+0000", 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5680000000000001e-06 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.682049+0000", 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1102062100000001 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.682191+0000", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014216599999999999 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.682499+0000", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00030818700000000002 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.689026+0000", 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006526558 2024-09-13T14:36:11.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705189+0000", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016163328000000001 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705221+0000", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1235000000000002e-05 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705446+0000", 2024-09-13T14:36:11.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000225186 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705472+0000", 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6550000000000002e-05 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705530+0000", 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7608000000000001e-05 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.339 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213733 6.17 6:ef052571:::benchmark_data_smithi017_185083_object89121:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.700096+0000", 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.565451932, 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0055427209999999, 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.340 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213733 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700096+0000", 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700096+0000", 2024-09-13T14:36:11.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700094+0000", 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700105+0000", 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1028e-05 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700118+0000", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3689999999999999e-05 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700126+0000", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.238e-06 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700187+0000", 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1177000000000001e-05 2024-09-13T14:36:11.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.700196+0000", 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.346e-06 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571864+0000", 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87166748500000002 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571868+0000", 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9380000000000002e-06 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.683861+0000", 2024-09-13T14:36:11.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1119936189999999 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.683980+0000", 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011875 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.684225+0000", 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024457899999999998 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.692055+0000", 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078307559999999995 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705236+0000", 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013181035000000001 2024-09-13T14:36:11.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.348 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705252+0000", 2024-09-13T14:36:11.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5224e-05 2024-09-13T14:36:11.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705558+0000", 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00030640399999999998 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705584+0000", 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5904000000000001e-05 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705638+0000", 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4623999999999997e-05 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213736 6.17 6:ea31f39e:::benchmark_data_smithi017_185083_object89124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.701354+0000", 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.564193332, 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.004405921, 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213736 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.351 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701354+0000", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701354+0000", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701353+0000", 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:11.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701360+0000", 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0029999999999997e-06 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701377+0000", 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7122e-05 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701384+0000", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3760000000000002e-06 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701447+0000", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2910000000000006e-05 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.701456+0000", 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7509999999999997e-06 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571887+0000", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87043061899999996 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571890+0000", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4209999999999999e-06 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.685385+0000", 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1134950400000001 2024-09-13T14:36:11.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.685501+0000", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116014 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.685740+0000", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000239046 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.692064+0000", 2024-09-13T14:36:11.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0063241230000000001 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705267+0000", 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013202762 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705658+0000", 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00039061800000000002 2024-09-13T14:36:11.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705674+0000", 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6096e-05 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705698+0000", 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4114999999999999e-05 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705760+0000", 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2329000000000006e-05 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213753 6.17 6:ef9ceeea:::benchmark_data_smithi017_185083_object89141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.359 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.716275+0000", 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.54927233000001, 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.989784744, 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213753 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716275+0000", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716275+0000", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716274+0000", 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:11.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716280+0000", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6309999999999997e-06 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716294+0000", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3952e-05 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716300+0000", 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2430000000000002e-06 2024-09-13T14:36:11.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716358+0000", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7105999999999998e-05 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716370+0000", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1953000000000001e-05 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571908+0000", 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.85553869699999996 2024-09-13T14:36:11.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571912+0000", 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4180000000000001e-06 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.687549+0000", 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.115637167 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.687670+0000", 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121662 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.687910+0000", 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000239979 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.695766+0000", 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078559860000000006 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705795+0000", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010028827000000001 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705836+0000", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0664000000000002e-05 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705926+0000", 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0339000000000004e-05 2024-09-13T14:36:11.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705971+0000", 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4659000000000001e-05 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706060+0000", 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9149999999999999e-05 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213754 6.17 6:ea618bf9:::benchmark_data_smithi017_185083_object89142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.716317+0000", 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.54923101099999, 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9899196080000001, 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213754 2024-09-13T14:36:11.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716317+0000", 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716317+0000", 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716316+0000", 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716319+0000", 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1429999999999998e-06 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716326+0000", 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.579e-06 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716330+0000", 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6160000000000004e-06 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716394+0000", 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3998000000000006e-05 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.716400+0000", 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9529999999999996e-06 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571930+0000", 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.85552965400000003 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571933+0000", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2329999999999998e-06 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.689057+0000", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.117123635 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.689122+0000", 2024-09-13T14:36:11.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5252000000000005e-05 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.689301+0000", 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000179106 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.695770+0000", 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064687909999999998 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705866+0000", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010096376000000001 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705896+0000", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9167999999999999e-05 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706112+0000", 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021647800000000001 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706147+0000", 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4480000000000002e-05 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706236+0000", 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9754999999999997e-05 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.376 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213810 6.17 6:ec19f0a3:::benchmark_data_smithi017_185083_object89198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.775980+0000", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.489567545, 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9305414010000002, 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213810 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775980+0000", 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775980+0000", 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775978+0000", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775984+0000", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7570000000000001e-06 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775993+0000", 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9549999999999998e-06 2024-09-13T14:36:11.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.775998+0000", 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9880000000000004e-06 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.776102+0000", 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010429699999999999 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.776113+0000", 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0135e-05 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571951+0000", 2024-09-13T14:36:11.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79583874399999999 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571955+0000", 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4309999999999998e-06 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.690494+0000", 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.118538885 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.690611+0000", 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117827 2024-09-13T14:36:11.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.690854+0000", 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024291699999999999 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.699476+0000", 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0086218330000000006 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706089+0000", 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066127060000000003 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706272+0000", 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018343600000000001 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706364+0000", 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1405999999999999e-05 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706411+0000", 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7250999999999999e-05 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706522+0000", 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011053299999999999 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213819 6.17 6:eb22f5ea:::benchmark_data_smithi017_185083_object89207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.386 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.784089+0000", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.481458664, 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9226488939999999, 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213819 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784089+0000", 2024-09-13T14:36:11.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784089+0000", 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784087+0000", 2024-09-13T14:36:11.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:11.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784095+0000", 2024-09-13T14:36:11.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5900000000000002e-06 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784109+0000", 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3866000000000001e-05 2024-09-13T14:36:11.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784116+0000", 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8120000000000002e-06 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784176+0000", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9957999999999998e-05 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.784185+0000", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5100000000000004e-06 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571973+0000", 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.78778826400000002 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571977+0000", 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3400000000000002e-06 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.692108+0000", 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.120131475 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.692231+0000", 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012264099999999999 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.692558+0000", 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032758700000000001 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.699483+0000", 2024-09-13T14:36:11.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069249009999999998 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706304+0000", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068209919999999997 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706336+0000", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2011999999999998e-05 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706564+0000", 2024-09-13T14:36:11.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022787799999999999 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706606+0000", 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1369999999999999e-05 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706738+0000", 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013234499999999999 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.396 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213834 6.17 6:ef5a73d0:::benchmark_data_smithi017_185083_object89222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.799055+0000", 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.466492644, 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9079717600000001, 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213834 2024-09-13T14:36:11.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799055+0000", 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799055+0000", 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799053+0000", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799065+0000", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1385e-05 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799082+0000", 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7914000000000002e-05 2024-09-13T14:36:11.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799090+0000", 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0639999999999996e-06 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799135+0000", 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5562000000000002e-05 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.799143+0000", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6830000000000008e-06 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571995+0000", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.77285216400000001 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.571998+0000", 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.484e-06 2024-09-13T14:36:11.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.694239+0000", 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1222403780000001 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.694317+0000", 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8387999999999996e-05 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.694560+0000", 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000243172 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.704957+0000", 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010396225 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706773+0000", 2024-09-13T14:36:11.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001816097 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706799+0000", 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5913000000000001e-05 2024-09-13T14:36:11.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706861+0000", 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2522999999999998e-05 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706930+0000", 2024-09-13T14:36:11.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9193e-05 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707027+0000", 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6498999999999999e-05 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213857 6.17 6:ea98cf8b:::benchmark_data_smithi017_185083_object89245:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.827653+0000", 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.43789508500001, 2024-09-13T14:36:11.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8795599250000001, 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213857 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827653+0000", 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827653+0000", 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827651+0000", 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827659+0000", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7340000000000008e-06 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827674+0000", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4653e-05 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827681+0000", 2024-09-13T14:36:11.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6329999999999996e-06 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827745+0000", 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4030999999999997e-05 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.827758+0000", 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2877999999999999e-05 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572016+0000", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74425823000000002 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572020+0000", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3000000000000002e-06 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.695808+0000", 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1237878640000001 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.695933+0000", 2024-09-13T14:36:11.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000125248 2024-09-13T14:36:11.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.696182+0000", 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024875400000000001 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.704961+0000", 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0087794369999999993 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.706836+0000", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001875102 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707057+0000", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000221432 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707083+0000", 2024-09-13T14:36:11.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5786e-05 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707120+0000", 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7107999999999998e-05 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707213+0000", 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2126000000000005e-05 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.417 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:213990 6.17 6:e93cc326:::benchmark_data_smithi017_185083_object89378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:58.984521+0000", 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.28102662699999, 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.72292885, 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 213990 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.418 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984521+0000", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984521+0000", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984520+0000", 2024-09-13T14:36:11.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984525+0000", 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6630000000000001e-06 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984536+0000", 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0896999999999999e-05 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984540+0000", 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3170000000000003e-06 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984574+0000", 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4375999999999997e-05 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:58.984580+0000", 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6099999999999997e-06 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572038+0000", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.58745747299999995 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572041+0000", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.388e-06 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.698023+0000", 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1259817649999999 2024-09-13T14:36:11.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.698112+0000", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8894999999999995e-05 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.698322+0000", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021075999999999999 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705125+0000", 2024-09-13T14:36:11.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068022150000000003 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707242+0000", 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021176229999999999 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707295+0000", 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2803000000000002e-05 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707321+0000", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6471000000000001e-05 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707358+0000", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7014000000000002e-05 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707450+0000", 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1436999999999999e-05 2024-09-13T14:36:11.425 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:214057 6.17 6:ed3927e1:::benchmark_data_smithi017_185083_object89445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:59.066785+0000", 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.19876260300001, 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.640897547, 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.426 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 214057 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066785+0000", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066785+0000", 2024-09-13T14:36:11.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066784+0000", 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066788+0000", 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2579999999999997e-06 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066797+0000", 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2169999999999994e-06 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066801+0000", 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1910000000000002e-06 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066855+0000", 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4453000000000002e-05 2024-09-13T14:36:11.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.066862+0000", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0960000000000003e-06 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572059+0000", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.505196533 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572062+0000", 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2940000000000001e-06 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.699513+0000", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.127450721 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.699586+0000", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2685e-05 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.699751+0000", 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016489799999999999 2024-09-13T14:36:11.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705129+0000", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0053782960000000003 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707269+0000", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021398820000000001 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707478+0000", 2024-09-13T14:36:11.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00020974399999999999 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707556+0000", 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7298000000000004e-05 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707592+0000", 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6477999999999999e-05 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707683+0000", 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0388000000000003e-05 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:214108 6.17 6:ee06b0dc:::benchmark_data_smithi017_185083_object89496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:59.159754+0000", 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 132.10579319799999, 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5506217310000001, 2024-09-13T14:36:11.434 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 214108 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159754+0000", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159754+0000", 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159754+0000", 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159758+0000", 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4460000000000003e-06 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159769+0000", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0829000000000001e-05 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159773+0000", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.245e-06 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159821+0000", 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7877999999999998e-05 2024-09-13T14:36:11.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.159828+0000", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0859999999999996e-06 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572080+0000", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.41225164399999997 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572083+0000", 2024-09-13T14:36:11.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3500000000000001e-06 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.700814+0000", 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.128730845 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.700965+0000", 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015125699999999999 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.701216+0000", 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025076900000000002 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705133+0000", 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039164559999999996 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707504+0000", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023716869999999999 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.707530+0000", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5537e-05 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.710225+0000", 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026956530000000001 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.710273+0000", 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7463999999999998e-05 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.710376+0000", 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103342 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:214154 6.17 6:ef2d908f:::benchmark_data_smithi017_185083_object89542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:59.313931+0000", 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 131.951617067, 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3995010620000001, 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 214154 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313931+0000", 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313931+0000", 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:11.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313929+0000", 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313935+0000", 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1260000000000003e-06 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313948+0000", 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2104e-05 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.313953+0000", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2240000000000003e-06 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.314009+0000", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6439e-05 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.314020+0000", 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1090999999999999e-05 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572095+0000", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25807497299999999 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:59.572099+0000", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.472e-06 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.702998+0000", 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1308994139999999 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.703110+0000", 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011154600000000001 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3,5", 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.703332+0000", 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000222785 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:11.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.705170+0000", 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0018377879999999999 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.710193+0000", 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005023128 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.710411+0000", 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000218023 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.713213+0000", 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028014289999999998 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.713308+0000", 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5198999999999994e-05 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:11.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:34:00.713432+0000", 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012363200000000001 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:11.451 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:11.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1526358018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:11.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:11.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 93473 93457 95.7356 64.1875 0.0155379 0.0104394 2024-09-13T14:36:11.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:11.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:11.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:11.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:11.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:11.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:11.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:11.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:12.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:12.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:12.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:12.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:12.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:12.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:12.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: pgmap v719: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:12.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:12.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: pgmap v719: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: pgmap v719: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:36:12.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:12.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:12.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:36:12.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:12.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:12.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:12.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:12.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 94514 94498 95.2408 65.0625 0.021101 0.0104943 2024-09-13T14:36:12.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:12.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:12.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:12.794 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:12.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:12.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:12.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:12.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:12.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:13.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:13.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:13.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:13.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:13.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:13.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:13.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:13.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:13.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 95435 95419 94.6427 57.5625 0.00696363 0.010561 2024-09-13T14:36:13.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:13.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:13.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:13.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:13.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:13.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:13.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:14.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:14.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:14.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:14.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:14.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:14.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:14.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:14.368 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:14 smithi189 ceph-mon[90992]: pgmap v721: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:14.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:36:14.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:14.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:14 smithi017 ceph-mon[103001]: pgmap v721: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:14.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:14 smithi017 ceph-mon[107556]: pgmap v721: 99 pgs: 99 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:14.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:14.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:14.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 96210 96194 93.9177 48.4375 0.00903009 0.0106412 2024-09-13T14:36:14.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:96266 7.0 7:02b293eb:::benchmark_data_smithi017_220393_object96265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:14.631206+0000", 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0040269590000000001, 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040494429999999998, 2024-09-13T14:36:14.636 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 96266 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631206+0000", 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.637 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631206+0000", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631204+0000", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631262+0000", 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7293999999999997e-05 2024-09-13T14:36:14.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631277+0000", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5738000000000001e-05 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631283+0000", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0700000000000003e-06 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631338+0000", 2024-09-13T14:36:14.639 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4341e-05 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631471+0000", 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013328600000000001 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.631595+0000", 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000124373 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:14.640 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:14.634711+0000", 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031158510000000002 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 1 2024-09-13T14:36:14.641 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:14.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:14.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:14.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:14.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:14.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:14.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:15.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:15.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:15.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:15.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:15.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:15.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:15.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:15.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:15.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:15.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:15.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 97197 97181 93.4218 61.6875 0.00809513 0.0106986 2024-09-13T14:36:15.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:15.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:15.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:15.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:15.845 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:15.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:15.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:36:15.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:15.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:15.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:16.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:16.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:16.159 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:36:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:16 smithi017 ceph-mon[103001]: pgmap v722: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:36:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:16 smithi017 ceph-mon[107556]: pgmap v722: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:36:16.225 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:16.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:16.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:16.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:16.298 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:36:16.299 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:36:16.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:36:16.335 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:36:16.369 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:36:16.369 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:36:16.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:16.395 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:16 smithi189 ceph-mon[90992]: pgmap v722: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:36:16.395 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:36:16.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:36:16.409 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:36:16.417 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:16.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:16.449 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:36:16.449 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:36:16.485 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:36:16.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:36:16.519 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:36:16.519 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:36:16.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:16.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:16.558 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:36:16.558 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:36:16.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 98070 98054 92.833 54.5625 0.0188347 0.0107662 2024-09-13T14:36:16.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:16.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:16.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:16.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:16.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:16.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:16.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:16.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:17.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:17.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:17.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:36:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[103001]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:36:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/436850790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:36:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[107556]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:36:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/436850790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:17.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:17.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:17.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:17.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:17.462 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/494204506' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:36:17.462 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:17 smithi189 ceph-mon[90992]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:36:17.462 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/436850790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:17.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:17.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:17.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:17.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 99091 99075 92.3998 63.8125 0.0159208 0.0108171 2024-09-13T14:36:17.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:17.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:17.695 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:17] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-09-13T14:36:17.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:17.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:17.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:17.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:17.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:18.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:18.055 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:18.055 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:18.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:18.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:18.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:18.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:18.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:18.397 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:18 smithi189 ceph-mon[90992]: pgmap v724: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:36:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:18 smithi017 ceph-mon[103001]: pgmap v724: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:36:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:18 smithi017 ceph-mon[107556]: pgmap v724: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-13T14:36:18.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:18.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:36:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:18.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 100065 100049 91.9357 60.875 0.0143746 0.0108719 2024-09-13T14:36:18.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:18.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:18.746 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:36:18.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:18.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:18.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:18.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:18.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:19.060 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:19.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:19.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:19.099 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:19.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:19.224 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:19.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:19.359 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:19.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:19.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:19.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 100956 100940 91.4104 55.6875 0.034251 0.0109339 2024-09-13T14:36:19.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:36:19.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:19.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:19.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:19.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:19.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:19.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:19.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:19.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:20.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:20.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:20.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:20.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:20.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:20.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:20.327 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:20 smithi189 ceph-mon[90992]: pgmap v725: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:36:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:20.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:20 smithi017 ceph-mon[103001]: pgmap v725: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:36:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:20 smithi017 ceph-mon[107556]: pgmap v725: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:36:20.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:20.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:20.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 101833 101817 90.8876 54.8125 0.0257051 0.0109976 2024-09-13T14:36:20.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:20.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:20.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:20.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:20.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:20.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:20.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:21.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:21.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:21.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:21.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:21.162 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:21.165 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:21.165 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:21.192 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:36:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/664021250' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:21.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:21.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:21.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/664021250' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/664021250' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:36:21.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:21.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 102822 102806 90.4781 61.8125 0.0116025 0.0110473 2024-09-13T14:36:21.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:21.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:21.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:21.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:21.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:21.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:21.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:36:21.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:22.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:22.124 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:22.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:22.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[103001]: pgmap v726: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:36:22.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:22.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:36:22.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[103001]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:36:22.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[107556]: pgmap v726: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:36:22.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:22.263 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:22.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:22.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:22.438 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:22 smithi189 ceph-mon[90992]: pgmap v726: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:36:22.438 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:22 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:22.438 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:36:22.438 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:22 smithi189 ceph-mon[90992]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:36:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:36:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:22 smithi017 ceph-mon[107556]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:36:22.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:22.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:22.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 103726 103710 90.0061 56.5 0.0129714 0.0111041 2024-09-13T14:36:22.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:22.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:22.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:22.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:22.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:22.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:22.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:22.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:23.028 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:23.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:23.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:23.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:23.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:23.206 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:36:23.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:23.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:23.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:23 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3227637130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:23 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3227637130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:23.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:23 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3227637130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:36:23.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:23.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 104646 104630 89.5608 57.5 0.0165017 0.01116 2024-09-13T14:36:23.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:23.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:23.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:23.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:23.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:23.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:23.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:23.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:24.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:24.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:24.157 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:24.157 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:24.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:24.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:24.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:24.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[103001]: pgmap v728: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:36:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:36:24.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[103001]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:36:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[107556]: pgmap v728: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:36:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:36:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:24 smithi017 ceph-mon[107556]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:36:24.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:24.501 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:24 smithi189 ceph-mon[90992]: pgmap v728: 99 pgs: 99 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:36:24.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:24 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:36:24.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:24 smithi189 ceph-mon[90992]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:36:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:24.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:24.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 105615 105599 89.1689 60.5625 0.00494861 0.0112055 2024-09-13T14:36:24.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:24.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:24.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:24.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:24.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:36:24.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:24.926 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:24.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:24.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:36:25.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:25.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:25.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:25.184 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:25.192 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:25.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:36:25.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:25.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:25.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:25.465 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:25.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:25.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 106471 106455 88.6933 53.5 0.020673 0.0112689 2024-09-13T14:36:25.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:25.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:25.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:25.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:25.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:25.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:25.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:25.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:26.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:26.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:26.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:26.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:26.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:26.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:26.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:26 smithi189 ceph-mon[90992]: pgmap v730: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:36:26.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:26 smithi017 ceph-mon[103001]: pgmap v730: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:36:26.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:26 smithi017 ceph-mon[107556]: pgmap v730: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:36:26.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 107244 107228 88.162 48.3125 0.0206816 0.0113375 2024-09-13T14:36:26.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:26.624 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:26.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:26.691 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:26.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:26.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:26.781 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:36:26.781 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:36:26.818 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:36:26.818 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:36:26.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:36:26.850 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:36:26.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:26.883 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:36:26.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:36:26.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:26.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:26.915 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:36:26.915 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:36:26.949 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:36:26.950 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:36:26.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:36:26.984 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:36:27.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:27.017 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:36:27.017 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:36:27.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:27.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:27.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:27.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:27.286 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:27 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3491353212' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:27.286 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:27.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:27.408 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:27.408 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:36:27.408 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:36:27.408 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:36:27.408 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:27.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:27.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:27.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:27.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:27.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 107959 107943 87.5974 44.6875 0.0101396 0.0114098 2024-09-13T14:36:27.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:27.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:27.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:27] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-13T14:36:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:27 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3491353212' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:27 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3491353212' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:27.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:27.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:27.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:27.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:27.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:28.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:28.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:28.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:28.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:28.278 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:28.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:28.380 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:28.411 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:28.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:28 smithi189 ceph-mon[90992]: pgmap v731: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:36:28.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:28.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:36:28.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:28.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 108853 108837 87.1907 55.875 0.0117106 0.0114633 2024-09-13T14:36:28.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:28.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:28 smithi017 ceph-mon[103001]: pgmap v731: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:36:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:28 smithi017 ceph-mon[107556]: pgmap v731: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:36:28.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:28.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:28.897 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:28.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:28.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:28.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:29.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:29.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:29.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:29.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:29.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:29.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:29.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:29.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:29.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:29.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:29.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:29.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 109795 109779 86.8322 58.875 0.00742051 0.0115114 2024-09-13T14:36:29.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:29.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:29.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:29.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:29.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:29.954 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:29.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:29.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:30.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:30.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:30.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:30.221 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:30.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:30.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:30.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:30.455 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:30.456 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:30 smithi189 ceph-mon[90992]: pgmap v732: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:36:30.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:30.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:30 smithi017 ceph-mon[103001]: pgmap v732: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:36:30.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:30 smithi017 ceph-mon[107556]: pgmap v732: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:36:30.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:36:30.572758+0000 min lat: 0.00129139 max lat: 0.133503 avg lat: 0.0115595 2024-09-13T14:36:30.573 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:36:30.573 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 110715 110699 86.4656 57.5 0.0144617 0.0115595 2024-09-13T14:36:30.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:30.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179068 5.11 5.50a6af91 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.735 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 508.16997675800002, 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80406319500000001, 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179068 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:36:30.736 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562185+0000", 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562183+0000", 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.737 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562189+0000", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4139999999999999e-06 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562203+0000", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3096e-05 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562209+0000", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4710000000000004e-06 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.738 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562251+0000", 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2101999999999998e-05 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.562258+0000", 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8499999999999996e-06 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366237+0000", 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80397893600000003 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366248+0000", 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1025e-05 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179072 5.11 5.a9629bd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 508.16879213300001, 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80290814399999999, 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.740 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179072 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563369+0000", 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.741 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563368+0000", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563373+0000", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9019999999999998e-06 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563385+0000", 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1528e-05 2024-09-13T14:36:30.742 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563390+0000", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7400000000000001e-06 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563593+0000", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000202559 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.563599+0000", 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366269+0000", 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80267021500000002 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366277+0000", 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6750000000000008e-06 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179268 5.11 5.957b59d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.744 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.94351152600001, 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.57765650000000002, 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179268 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.745 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788650+0000", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788648+0000", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788655+0000", 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7329999999999997e-06 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.746 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788668+0000", 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.296e-05 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.788673+0000", 2024-09-13T14:36:30.747 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7089999999999999e-06 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.798794+0000", 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010120891999999999 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.798801+0000", 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6980000000000001e-06 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366298+0000", 2024-09-13T14:36:30.749 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.56749667100000001 2024-09-13T14:36:30.749 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.749 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.749 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.749 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366306+0000", 2024-09-13T14:36:30.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:30.750 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6989999999999992e-06 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179352 5.11 5.b7d57211 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.83664165699997, 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.47084923899999997, 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.751 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179352 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895520+0000", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895518+0000", 2024-09-13T14:36:30.752 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895523+0000", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4399999999999998e-06 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895531+0000", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5399999999999996e-06 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.895536+0000", 2024-09-13T14:36:30.753 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7040000000000002e-06 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.897591+0000", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002054967 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.897598+0000", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6109999999999999e-06 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366360+0000", 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.46876190000000001 2024-09-13T14:36:30.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366369+0000", 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.499e-06 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179409 5.11 5.eddc87b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.77307156699999, 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40730723800000002, 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.755 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179409 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959090+0000", 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.756 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959088+0000", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959096+0000", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9650000000000008e-06 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959107+0000", 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0278999999999999e-05 2024-09-13T14:36:30.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.959112+0000", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3750000000000002e-06 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960225+0000", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011133849999999999 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960278+0000", 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2590000000000003e-05 2024-09-13T14:36:30.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366391+0000", 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40611253200000003 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366397+0000", 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5729999999999996e-06 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179414 5.11 5.998ec5d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.77128243599998, 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40555330499999998, 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179414 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.760 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960879+0000", 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960878+0000", 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.761 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960883+0000", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5179999999999998e-06 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960895+0000", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2782000000000001e-05 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.960901+0000", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0649999999999998e-06 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.961050+0000", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014901500000000001 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.961055+0000", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2059999999999998e-06 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366423+0000", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40536831699999998 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366432+0000", 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3070000000000008e-06 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179424 5.11 5.a52315d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.75601308500001, 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.39031132600000001, 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.764 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179424 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976148+0000", 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976147+0000", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976153+0000", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4649999999999999e-06 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976164+0000", 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0903e-05 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976170+0000", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8599999999999998e-06 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976365+0000", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000194673 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:02.976371+0000", 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9120000000000003e-06 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366453+0000", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.39008283999999999 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366460+0000", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3090000000000001e-06 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179473 5.11 5.b26654d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:36:30.768 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.67470481200002, 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30902944900000001, 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179473 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057457+0000", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057456+0000", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057462+0000", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8989999999999998e-06 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057472+0000", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.077e-05 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057478+0000", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0859999999999998e-06 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057676+0000", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019763699999999999 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.057682+0000", 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1310000000000003e-06 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366479+0000", 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30879694600000002 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366486+0000", 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9789999999999996e-06 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.772 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:179488 5.11 5.79892fd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e124)", 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 507.65374521799998, 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28810034800000001, 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179488 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.773 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078416+0000", 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:30.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078423+0000", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5650000000000004e-06 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078436+0000", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.324e-05 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078442+0000", 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4079999999999997e-06 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078806+0000", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036393999999999999 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.078812+0000", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1800000000000001e-06 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366506+0000", 2024-09-13T14:36:30.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.287694279 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:03.366517+0000", 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0479000000000001e-05 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194100 5.11 5:899dff21:::benchmark_data_smithi017_147622_object59471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:36:30.778 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 487.244849278, 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93940912399999998, 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194100 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487312+0000", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487310+0000", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487320+0000", 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8600000000000005e-06 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487332+0000", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2334000000000001e-05 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487338+0000", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.049e-06 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487503+0000", 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016442900000000001 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.487511+0000", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421662+0000", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93415139400000002 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421760+0000", 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8265999999999997e-05 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.421840+0000", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9374000000000003e-05 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426549+0000", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047089769999999996 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426575+0000", 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6684000000000001e-05 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426589+0000", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4156e-05 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426686+0000", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6089999999999996e-05 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426721+0000", 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5760000000000003e-05 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194109 5.11 5:8fec395f:::benchmark_data_smithi017_147622_object59480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 487.23111153000002, 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.93220517000000003, 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194109 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.785 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501050+0000", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501048+0000", 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501056+0000", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4189999999999996e-06 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501073+0000", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7646e-05 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501081+0000", 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6510000000000001e-06 2024-09-13T14:36:30.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501133+0000", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1722999999999999e-05 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.501155+0000", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1834000000000001e-05 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423537+0000", 2024-09-13T14:36:30.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92238267699999998 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423686+0000", 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000148648 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.423774+0000", 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8084999999999994e-05 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.789 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.428865+0000", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0050907490000000003 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433082+0000", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042173330000000002 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433133+0000", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1220000000000001e-05 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433218+0000", 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4188000000000001e-05 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433255+0000", 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7561999999999997e-05 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194177 5.11 5:8c2311cb:::benchmark_data_smithi017_147622_object59548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 487.16601165100002, 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86727486499999995, 2024-09-13T14:36:30.791 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194177 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.792 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566150+0000", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566148+0000", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566154+0000", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0589999999999999e-06 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566168+0000", 2024-09-13T14:36:30.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3652000000000001e-05 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566174+0000", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5260000000000001e-06 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566217+0000", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3430000000000003e-05 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.566224+0000", 2024-09-13T14:36:30.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2420000000000003e-06 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425124+0000", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.858899682 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425232+0000", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010812899999999999 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.425339+0000", 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010717699999999999 2024-09-13T14:36:30.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433108+0000", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077691629999999999 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433327+0000", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000218097 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433334+0000", 2024-09-13T14:36:30.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8229999999999995e-06 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433380+0000", 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5207000000000003e-05 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433425+0000", 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5126999999999997e-05 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194205 5.11 5:8ab11436:::benchmark_data_smithi017_147622_object59576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 487.136008527, 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84416819300000001, 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194205 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.798 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596153+0000", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596151+0000", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596158+0000", 2024-09-13T14:36:30.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1779999999999997e-06 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596172+0000", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3178e-05 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596177+0000", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5999999999999997e-06 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596216+0000", 2024-09-13T14:36:30.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9270999999999997e-05 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.596225+0000", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2139999999999996e-06 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426749+0000", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83052429999999999 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426843+0000", 2024-09-13T14:36:30.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3908999999999998e-05 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.426946+0000", 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103627 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433290+0000", 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006343591 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.802 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433477+0000", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018701699999999999 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440230+0000", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006752699 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440282+0000", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2042000000000001e-05 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440321+0000", 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9345999999999999e-05 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194221 5.11 5:8ecddf2c:::benchmark_data_smithi017_147622_object59592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 487.12048193800001, 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83324189699999995, 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.804 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194221 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611680+0000", 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611678+0000", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611689+0000", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0625000000000001e-05 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611700+0000", 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0900999999999999e-05 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.806 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611706+0000", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0809999999999999e-06 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611749+0000", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3550000000000001e-05 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.611758+0000", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.918e-06 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.428926+0000", 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.81716769499999997 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.429052+0000", 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012662799999999999 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.429167+0000", 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000114589 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440185+0000", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011018035000000001 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440329+0000", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000143702 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444827+0000", 2024-09-13T14:36:30.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0044977840000000003 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444882+0000", 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5121999999999998e-05 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444921+0000", 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9728999999999999e-05 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194323 5.11 5:8858a9dc:::benchmark_data_smithi017_147622_object59694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.99792480899998, 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71087829400000002, 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194323 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734237+0000", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734235+0000", 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734242+0000", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4599999999999997e-06 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734260+0000", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7839e-05 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734267+0000", 2024-09-13T14:36:30.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.092e-06 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734313+0000", 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6047000000000001e-05 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.734322+0000", 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9949999999999994e-06 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431146+0000", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69682316600000005 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431247+0000", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101891 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.431358+0000", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000110159 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440214+0000", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0088563949999999995 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440332+0000", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118144 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445019+0000", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046870799999999997 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445076+0000", 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7064000000000001e-05 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445115+0000", 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8634999999999999e-05 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194411 5.11 5:8901ba5a:::benchmark_data_smithi017_147622_object59782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:36:30.817 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.89231913600003, 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.60801552800000003, 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194411 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839842+0000", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839841+0000", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839849+0000", 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4710000000000007e-06 2024-09-13T14:36:30.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839867+0000", 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7567e-05 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839874+0000", 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1130000000000003e-06 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839941+0000", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7269999999999998e-05 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.839950+0000", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8890000000000008e-06 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433500+0000", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.59354982199999995 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433593+0000", 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3528000000000003e-05 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.433690+0000", 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6317999999999995e-05 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440359+0000", 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066693330000000004 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442529+0000", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021701609999999999 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447702+0000", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0051729669999999997 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447810+0000", 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010777699999999999 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447858+0000", 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8075000000000003e-05 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194483 5.11 5:8deb5587:::benchmark_data_smithi017_147622_object59854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.79579654899999, 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51168034699999998, 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.824 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194483 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936365+0000", 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936363+0000", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936371+0000", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2179999999999999e-06 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936386+0000", 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4722000000000001e-05 2024-09-13T14:36:30.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936404+0000", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7702000000000001e-05 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936451+0000", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7008000000000001e-05 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:23.936458+0000", 2024-09-13T14:36:30.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9209999999999992e-06 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435539+0000", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.49908090999999999 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435637+0000", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7547999999999995e-05 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.435734+0000", 2024-09-13T14:36:30.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7188000000000005e-05 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442568+0000", 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068333930000000001 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444770+0000", 2024-09-13T14:36:30.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022027959999999999 2024-09-13T14:36:30.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447888+0000", 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031180650000000002 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448006+0000", 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011807399999999999 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448045+0000", 2024-09-13T14:36:30.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8838000000000003e-05 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194542 5.11 5:891d791e:::benchmark_data_smithi017_147622_object59913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.71003511200001, 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.42990321399999998, 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.832 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194542 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022126+0000", 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022125+0000", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022132+0000", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1380000000000001e-06 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022150+0000", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8580000000000002e-05 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022158+0000", 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6930000000000007e-06 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022213+0000", 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4965e-05 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.022221+0000", 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6790000000000005e-06 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437665+0000", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.41544472399999999 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437763+0000", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7056e-05 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.437865+0000", 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010253 2024-09-13T14:36:30.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.442572+0000", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047068960000000003 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444968+0000", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023965150000000001 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.451923+0000", 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069545290000000001 2024-09-13T14:36:30.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.451980+0000", 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7089999999999997e-05 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452030+0000", 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9486e-05 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.838 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194600 5.11 5:8be3cac2:::benchmark_data_smithi017_147622_object59971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.64021836199998, 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.36026992099999999, 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194600 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091943+0000", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091941+0000", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091951+0000", 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0305999999999999e-05 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091974+0000", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2739e-05 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.091982+0000", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2889999999999998e-06 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.092045+0000", 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3238000000000007e-05 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.092053+0000", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8699999999999992e-06 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440389+0000", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34833567799999998 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440474+0000", 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5704000000000005e-05 2024-09-13T14:36:30.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.440580+0000", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000105441 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.444934+0000", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004354485 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.445181+0000", 2024-09-13T14:36:30.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024614100000000001 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452150+0000", 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069696760000000002 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452177+0000", 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7010999999999999e-05 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452213+0000", 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5794000000000003e-05 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35034.0:194672 5.11 5:896b780c:::benchmark_data_smithi017_147622_object60043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 486.52841975299998, 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24859889499999999, 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35034", 2024-09-13T14:36:30.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 194672 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203742+0000", 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203741+0000", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203747+0000", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9450000000000004e-06 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203761+0000", 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3733999999999999e-05 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203768+0000", 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5799999999999997e-06 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203816+0000", 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8355e-05 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:36:30.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.203834+0000", 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8175000000000001e-05 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443084+0000", 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.23924999799999999 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443204+0000", 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011947699999999999 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.443324+0000", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011992199999999999 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.447175+0000", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038516660000000001 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.448258+0000", 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001082246 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452258+0000", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004000417 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452306+0000", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7936999999999999e-05 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:28:24.452341+0000", 2024-09-13T14:36:30.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4644000000000002e-05 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:30.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:30.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:30.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:30.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:31.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:36:31.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:31.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:31.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:31.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:31.423 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:31.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:31.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:31.573 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 111607 111591 86.0854 55.75 0.0165261 0.0116103 2024-09-13T14:36:31.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:31.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:31.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:31.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:31.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:31.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:31.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:31.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:32.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:32.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:32.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:32.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:32.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:32.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:32.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:32 smithi189 ceph-mon[90992]: pgmap v733: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:36:32.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:36:32.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:32 smithi017 ceph-mon[103001]: pgmap v733: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:36:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:32 smithi017 ceph-mon[107556]: pgmap v733: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:36:32.490 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:32.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:32.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 112395 112379 85.633 49.25 0.0119392 0.0116716 2024-09-13T14:36:32.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:32.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:32.801 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:32.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:32.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:32.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:32.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:33.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:33.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:33.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:33.261 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:36:33.263 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:36:33.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:33.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:33.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:33.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:33.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1294633754' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3806257906' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1294633754' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3806257906' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1294633754' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3806257906' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:36:33.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:33.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 113236 113220 85.2345 52.5625 0.00974539 0.0117262 2024-09-13T14:36:33.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:33.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:33.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:33.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:33.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:33.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:33.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:33.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:33.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:33.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:34.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:34.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:34.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:36:34.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:34.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: pgmap v734: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:36:34.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:34 smithi189 ceph-mon[90992]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:36:34.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: pgmap v734: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[103001]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: pgmap v734: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:36:34.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:34.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:36:34.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:34 smithi017 ceph-mon[107556]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:36:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:34.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:34.576 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:34.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 114119 114103 84.8767 55.1875 0.0113169 0.011777 2024-09-13T14:36:34.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:34.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:34.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:34.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:34.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:34.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:34.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:34.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:34.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:35.006 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:35.036 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:114487 7.5 7:a67eff22:::benchmark_data_smithi017_220393_object114486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:34.997992+0000", 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.034077663000000001, 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.03410258, 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.088 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 114487 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.997992+0000", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.997992+0000", 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.997990+0000", 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998033+0000", 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2762999999999997e-05 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998050+0000", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6923e-05 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998056+0000", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.139e-06 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998108+0000", 2024-09-13T14:36:35.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2388000000000001e-05 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998265+0000", 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000156222 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:34.998423+0000", 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000158484 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.002371+0000", 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039474749999999998 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.018955+0000", 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016584575000000001 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:114494 7.5 7:abf139ee:::benchmark_data_smithi017_220393_object114493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-09-13T14:36:35.093 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.003144+0000", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.028925764999999999, 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.029040549999999998, 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 114494 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003144+0000", 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003144+0000", 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003142+0000", 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003194+0000", 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2034999999999999e-05 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003210+0000", 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5979000000000001e-05 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003217+0000", 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5660000000000003e-06 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003267+0000", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9824999999999998e-05 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003393+0000", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012583700000000001 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.003530+0000", 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013753100000000001 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.010931+0000", 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074012440000000004 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019004+0000", 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080725610000000007 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.098 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:114505 7.5 7:a7404909:::benchmark_data_smithi017_220393_object114504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.019096+0000", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.012974006, 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013142109000000001, 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 114505 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019096+0000", 2024-09-13T14:36:35.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019096+0000", 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019094+0000", 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019135+0000", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0921999999999999e-05 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019148+0000", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3244999999999999e-05 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019154+0000", 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7729999999999996e-06 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019196+0000", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2018999999999999e-05 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019337+0000", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014111500000000001 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.019492+0000", 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015438600000000001 2024-09-13T14:36:35.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.022575+0000", 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030833459999999998 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.030885+0000", 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0083099279999999994 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:114507 7.5 7:ab2d3331:::benchmark_data_smithi017_220393_object114506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.023706+0000", 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0083634360000000001, 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.008626412, 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 114507 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023706+0000", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023706+0000", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023705+0000", 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023806+0000", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010126799999999999 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023829+0000", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2198e-05 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023836+0000", 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2400000000000001e-06 2024-09-13T14:36:35.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.023896+0000", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0096000000000002e-05 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.024025+0000", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000129271 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.024173+0000", 2024-09-13T14:36:35.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000147293 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.027223+0000", 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030503930000000002 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.031618+0000", 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043954450000000004 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.108 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.45566.0:114487 7.5 e263/247 ondisk, result = 0)", 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.032262+0000", 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4294967295.9998078, 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015423100000000001, 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032262+0000", 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032262+0000", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032256+0000", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999943 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032266+0000", 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6749999999999997e-06 2024-09-13T14:36:35.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032275+0000", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.781e-06 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032281+0000", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1019999999999996e-06 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032339+0000", 2024-09-13T14:36:35.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8968999999999997e-05 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032360+0000", 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0835999999999998e-05 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.45566.0:114494 7.5 e263/247)", 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.032290+0000", 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4294967295.9997797, 2024-09-13T14:36:35.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018772299999999999, 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032290+0000", 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032290+0000", 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032290+0000", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032292+0000", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6199999999999999e-06 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032298+0000", 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1009999999999997e-06 2024-09-13T14:36:35.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032302+0000", 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.456e-06 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032479+0000", 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017722199999999999 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.45566.0:114505 7.5 e263/247)", 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.032337+0000", 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4294967295.9997325, 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0001928, 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032337+0000", 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032337+0000", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032337+0000", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032341+0000", 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.676e-06 2024-09-13T14:36:35.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032381+0000", 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0238999999999999e-05 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032386+0000", 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6540000000000004e-06 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.45566.0:114507 7.5 e263/247)", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.032392+0000", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4294967295.9996777, 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018381800000000001, 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032392+0000", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032392+0000", 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032392+0000", 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032395+0000", 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.993e-06 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032399+0000", 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3869999999999997e-06 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032401+0000", 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4600000000000002e-06 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.45566.0:114511 7.6 e263/247 7:6bb8ba59:::benchmark_data_smithi017_220393_object114510:head v 263'7112, mlcod=263'7109)", 2024-09-13T14:36:35.121 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:36:35.032404+0000", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 4294967295.9996662, 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022668100000000001, 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032404+0000", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032404+0000", 2024-09-13T14:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032403+0000", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967296 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032428+0000", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4341e-05 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032434+0000", 2024-09-13T14:36:35.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0619999999999997e-06 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032437+0000", 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.428e-06 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032493+0000", 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6029999999999997e-05 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:36:35.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:36:35.032514+0000", 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1067999999999999e-05 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 9 2024-09-13T14:36:35.125 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:36:35.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:35.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:35.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:35.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:35.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:35.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:35.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:35.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:35.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 114955 114939 84.4927 52.25 0.0108458 0.0118298 2024-09-13T14:36:35.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:35.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:35.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:35.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:35.806 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:35.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:36:35.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:35.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:35.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:36.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:36.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:36.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:36.180 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:36.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:36.197 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:36:36.198 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:36:36.235 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:36:36.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:36:36.271 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:36:36.271 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:36:36.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:36.314 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:36:36.314 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:36:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:36.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:36.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:36:36.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:36:36.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:36:36.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:36:36.427 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:36:36.427 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:36:36.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:36.466 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:36:36.466 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:36:36.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:36.485 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:36 smithi017 ceph-mon[103001]: pgmap v736: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T14:36:36.485 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/455492358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:36.485 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:36 smithi017 ceph-mon[107556]: pgmap v736: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T14:36:36.485 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/455492358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:36.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 115778 115762 84.1083 51.4375 0.0173692 0.0118841 2024-09-13T14:36:36.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:36.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:36.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:36 smithi189 ceph-mon[90992]: pgmap v736: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T14:36:36.627 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/455492358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:36.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:36.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:36.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:36.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:36.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:36.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:37.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:36:37.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:37.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:37.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:37.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:37.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:37.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:37.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:37.526 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:37.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:37.577 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 116602 116586 83.7334 51.5 0.031751 0.0119368 2024-09-13T14:36:37.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:37.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:37] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-13T14:36:37.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:37.801 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:37.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:37.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:37.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:38.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:38.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:38.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:38.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:38.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:38.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:38.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:38 smithi017 ceph-mon[103001]: pgmap v737: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-13T14:36:38.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:38 smithi017 ceph-mon[107556]: pgmap v737: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-13T14:36:38.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:38 smithi189 ceph-mon[90992]: pgmap v737: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-09-13T14:36:38.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:38.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:38.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 117477 117461 83.4033 54.6875 0.0152982 0.0119843 2024-09-13T14:36:38.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:38.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:38.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:38.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:38.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:38.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:38.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:38.976 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:38.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:38.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:39.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:39.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:39.224 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:39.236 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:39.236 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:39.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:39.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:39.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:39.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:39.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:39.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 118302 118286 83.0455 51.5625 0.00935629 0.0120359 2024-09-13T14:36:39.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:39.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:39.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:39.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:36:39.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:39.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:39.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:40.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:40.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:40.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:40.217 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:40.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:40.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:40.268 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:36:40.285 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:36:40.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:40.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:40 smithi017 ceph-mon[103001]: pgmap v738: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:36:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:36:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:40 smithi017 ceph-mon[107556]: pgmap v738: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:36:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:36:40.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:36:40.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:40 smithi189 ceph-mon[90992]: pgmap v738: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:36:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:36:40.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0079 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 119131 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 82.7226 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 35.3724 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 215.188 2024-09-13T14:36:40.578 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 44.6875 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1323 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 565.958 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3443 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 715 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0120841 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.00849461 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.133503 2024-09-13T14:36:40.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00129139 2024-09-13T14:36:40.583 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:36:40.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:40.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:40.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:40.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:40.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:40.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:40.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:41.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:41.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:41.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:41.229 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:41.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.360 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:41.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:36:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:41 smithi017 ceph-mon[103001]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:36:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:36:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:41 smithi017 ceph-mon[107556]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:36:41.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:41.498 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:36:41.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:36:41.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/893291950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:36:41.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:41 smithi189 ceph-mon[90992]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:36:41.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:41.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:41.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:41.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:41.903 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:41.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:41.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:41.939 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:42.012 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:42.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:42.058 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:36:42.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:42.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:42.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:42.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:42.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:42.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[103001]: pgmap v740: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2024-09-13T14:36:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1854460996' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[107556]: pgmap v740: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1854460996' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:42 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:42 smithi189 ceph-mon[90992]: pgmap v740: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2024-09-13T14:36:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1854460996' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:42 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:36:42.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:42.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:42.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:42.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:42.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:42.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:42.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:42.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:36:42.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:43.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:43.120 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:36:43.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:43.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:43.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:43.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:43.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:43.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:43.467 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:43 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:36:43.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:43 smithi189 ceph-mon[90992]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:36:43.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:43.596 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:43.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:36:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[103001]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:36:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:36:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:43 smithi017 ceph-mon[107556]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:36:43.724 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:43.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:43.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:43.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:43.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:43.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:44.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:44.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:44.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:44.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:44.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:44.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:44.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:44.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:44.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:44.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:44.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:44.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:44 smithi017 ceph-mon[103001]: pgmap v741: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 104 B/s rd, 42 MiB/s wr, 780 op/s 2024-09-13T14:36:44.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:44 smithi017 ceph-mon[107556]: pgmap v741: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 104 B/s rd, 42 MiB/s wr, 780 op/s 2024-09-13T14:36:44.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:44.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:44 smithi189 ceph-mon[90992]: pgmap v741: 99 pgs: 99 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 104 B/s rd, 42 MiB/s wr, 780 op/s 2024-09-13T14:36:44.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:44.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:44.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:44.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:44.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:45.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:45.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:45.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:45.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:45.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:45.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:45.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:45.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:45.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:45.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:45.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:45.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:45.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:45.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:45.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:45.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:45.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:45.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:46.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:46.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:46.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:46.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:46.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:46.282 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:46.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:46.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:46.522 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:46 smithi017 ceph-mon[103001]: pgmap v743: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 990 op/s 2024-09-13T14:36:46.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:46 smithi017 ceph-mon[107556]: pgmap v743: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 990 op/s 2024-09-13T14:36:46.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:46.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:46.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:46.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:46 smithi189 ceph-mon[90992]: pgmap v743: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 44 MiB/s wr, 990 op/s 2024-09-13T14:36:46.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:46.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:46.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:46.980 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:46.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:47.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:47.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:47.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:47.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:47.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:47.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:47.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:47.437 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:47] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:36:47.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:36:47.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:36:47.552 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:36:47.552 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:36:47.558 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:36:47.559 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:36:47.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:47.602 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:36:47.602 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:36:47.639 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:36:47.639 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:36:47.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:36:47.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:47.680 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:36:47.680 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:36:47.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:47.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:47.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:47.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:47.880 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:36:47.881 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:36:47.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:48.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:48.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:48.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:48.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:48.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:48.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:48.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:48.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:48.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:48 smithi017 ceph-mon[103001]: pgmap v744: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 725 op/s 2024-09-13T14:36:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/540090716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:48 smithi017 ceph-mon[107556]: pgmap v744: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 725 op/s 2024-09-13T14:36:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/540090716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:48.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:48 smithi189 ceph-mon[90992]: pgmap v744: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 23 MiB/s wr, 725 op/s 2024-09-13T14:36:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/540090716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:36:48.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:48.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:48.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:48.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:48.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:49.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:49.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:49.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:49.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:49.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:49.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:49.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:49.489 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:49.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:49.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:49.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:49 smithi017 ceph-mon[107556]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:36:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:49 smithi017 ceph-mon[103001]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:36:49.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:49.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:49 smithi189 ceph-mon[90992]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:36:49.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:49.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:49.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:49.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:50.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:50.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:50.141 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:50.167 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:50.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:50.296 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:50.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:50.425 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:50 smithi189 ceph-mon[90992]: pgmap v745: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 116 B/s rd, 14 MiB/s wr, 858 op/s 2024-09-13T14:36:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:36:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:36:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:50.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:50.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:50.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:50 smithi017 ceph-mon[103001]: pgmap v745: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 116 B/s rd, 14 MiB/s wr, 858 op/s 2024-09-13T14:36:50.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:50 smithi017 ceph-mon[107556]: pgmap v745: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 116 B/s rd, 14 MiB/s wr, 858 op/s 2024-09-13T14:36:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:50.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:50.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:50.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:50.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:50.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:50.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:51.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:51.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:51.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:51.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:51.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:51.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:51.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:51.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:51.619 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:36:51.619 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:36:51.619 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:36:51.619 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:36:51.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:51.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:51.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:51.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:51.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:51.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:51.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:52.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:52.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:52.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:52.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:52.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:52.345 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:52.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:52.471 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:52.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:52.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:36:52.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:52.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:52 smithi017 ceph-mon[103001]: pgmap v746: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 854 op/s 2024-09-13T14:36:52.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:52 smithi017 ceph-mon[107556]: pgmap v746: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 854 op/s 2024-09-13T14:36:52.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:52.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:52 smithi189 ceph-mon[90992]: pgmap v746: 99 pgs: 99 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 854 op/s 2024-09-13T14:36:52.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:52.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:52.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:52.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:52.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:53.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:53.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:53.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:53.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:53.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:53.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:53.419 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:53.521 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:53.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:53.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:53.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:53.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:53.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:53.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:53.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:53.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:54.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:54.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:54.103 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:54.141 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":744,"stamp":"2024-09-13T14:36:47.051845+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7624333285,"num_objects":116294,"num_object_clones":0,"num_object_copies":348882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116294,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":122905,"num_write_kb":7637008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121130,"ondisk_log_size":121130,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":25148796,"kb_used_data":22429940,"kb_used_omap":36,"kb_used_meta":2718683,"kb_avail":724812420,"statfs":{"total":767960285184,"available":742207918080,"internally_reserved":0,"allocated":22968258560,"data_stored":22957843793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37497,"internal_metadata":2783931783},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":4718616,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5805,"num_write_kb":188033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005347"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":410,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.134453+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:36:43.134453+0000","last_peered":"2024-09-13T14:36:43.134453+0000","last_clean":"2024-09-13T14:36:43.134453+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:36:43.134453+0000","last_undegraded":"2024-09-13T14:36:43.134453+0000","last_fullsized":"2024-09-13T14:36:43.134453+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":230,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263551+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:36:33.263551+0000","last_peered":"2024-09-13T14:36:33.263551+0000","last_clean":"2024-09-13T14:36:33.263551+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:36:33.263551+0000","last_undegraded":"2024-09-13T14:36:33.263551+0000","last_fullsized":"2024-09-13T14:36:33.263551+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272299+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:36:40.272299+0000","last_peered":"2024-09-13T14:36:40.272299+0000","last_clean":"2024-09-13T14:36:40.272299+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:36:40.272299+0000","last_undegraded":"2024-09-13T14:36:40.272299+0000","last_fullsized":"2024-09-13T14:36:40.272299+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":393,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282509+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:36:40.282509+0000","last_peered":"2024-09-13T14:36:40.282509+0000","last_clean":"2024-09-13T14:36:40.282509+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:36:40.282509+0000","last_undegraded":"2024-09-13T14:36:40.282509+0000","last_fullsized":"2024-09-13T14:36:40.282509+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":394,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282658+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:36:40.282658+0000","last_peered":"2024-09-13T14:36:40.282658+0000","last_clean":"2024-09-13T14:36:40.282658+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:36:40.282658+0000","last_undegraded":"2024-09-13T14:36:40.282658+0000","last_fullsized":"2024-09-13T14:36:40.282658+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263383+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:36:33.263383+0000","last_peered":"2024-09-13T14:36:33.263383+0000","last_clean":"2024-09-13T14:36:33.263383+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:36:33.263383+0000","last_undegraded":"2024-09-13T14:36:33.263383+0000","last_fullsized":"2024-09-13T14:36:33.263383+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":410,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.131505+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:36:43.131505+0000","last_peered":"2024-09-13T14:36:43.131505+0000","last_clean":"2024-09-13T14:36:43.131505+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:36:43.131505+0000","last_undegraded":"2024-09-13T14:36:43.131505+0000","last_fullsized":"2024-09-13T14:36:43.131505+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":236,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.257922+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:36:33.257922+0000","last_peered":"2024-09-13T14:36:33.257922+0000","last_clean":"2024-09-13T14:36:33.257922+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:36:33.257922+0000","last_undegraded":"2024-09-13T14:36:33.257922+0000","last_fullsized":"2024-09-13T14:36:33.257922+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"265'7680","reported_seq":7727,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:44.612816+0000","last_change":"2024-09-13T14:34:57.524042+0000","last_active":"2024-09-13T14:36:44.612816+0000","last_peered":"2024-09-13T14:36:44.612816+0000","last_clean":"2024-09-13T14:36:44.612816+0000","last_became_active":"2024-09-13T14:34:57.523748+0000","last_became_peered":"2024-09-13T14:34:57.523748+0000","last_unstale":"2024-09-13T14:36:44.612816+0000","last_undegraded":"2024-09-13T14:36:44.612816+0000","last_fullsized":"2024-09-13T14:36:44.612816+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7680,"log_dups_size":0,"ondisk_log_size":7680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:42.905684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473563136,"num_objects":7226,"num_object_clones":0,"num_object_copies":21678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7680,"num_write_kb":476992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":229,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263859+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:36:33.263859+0000","last_peered":"2024-09-13T14:36:33.263859+0000","last_clean":"2024-09-13T14:36:33.263859+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:36:33.263859+0000","last_undegraded":"2024-09-13T14:36:33.263859+0000","last_fullsized":"2024-09-13T14:36:33.263859+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271526+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:36:40.271526+0000","last_peered":"2024-09-13T14:36:40.271526+0000","last_clean":"2024-09-13T14:36:40.271526+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:36:40.271526+0000","last_undegraded":"2024-09-13T14:36:40.271526+0000","last_fullsized":"2024-09-13T14:36:40.271526+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"265'7742","reported_seq":7789,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:44.884065+0000","last_change":"2024-09-13T14:34:57.522300+0000","last_active":"2024-09-13T14:36:44.884065+0000","last_peered":"2024-09-13T14:36:44.884065+0000","last_clean":"2024-09-13T14:36:44.884065+0000","last_became_active":"2024-09-13T14:34:57.522005+0000","last_became_peered":"2024-09-13T14:34:57.522005+0000","last_unstale":"2024-09-13T14:36:44.884065+0000","last_undegraded":"2024-09-13T14:36:44.884065+0000","last_fullsized":"2024-09-13T14:36:44.884065+0000","mapping_epoch":247,"log_start":"252'1000","ondisk_log_start":"252'1000","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:56.018499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479592448,"num_objects":7318,"num_object_clones":0,"num_object_copies":21954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7742,"num_write_kb":481920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":247,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263792+0000","last_change":"2024-09-13T14:34:46.649053+0000","last_active":"2024-09-13T14:36:33.263792+0000","last_peered":"2024-09-13T14:36:33.263792+0000","last_clean":"2024-09-13T14:36:33.263792+0000","last_became_active":"2024-09-13T14:34:46.648725+0000","last_became_peered":"2024-09-13T14:34:46.648725+0000","last_unstale":"2024-09-13T14:36:33.263792+0000","last_undegraded":"2024-09-13T14:36:33.263792+0000","last_fullsized":"2024-09-13T14:36:33.263792+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:03.935325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":262,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256415+0000","last_change":"2024-09-13T14:34:46.594184+0000","last_active":"2024-09-13T14:36:33.256415+0000","last_peered":"2024-09-13T14:36:33.256415+0000","last_clean":"2024-09-13T14:36:33.256415+0000","last_became_active":"2024-09-13T14:34:46.593911+0000","last_became_peered":"2024-09-13T14:34:46.593911+0000","last_unstale":"2024-09-13T14:36:33.256415+0000","last_undegraded":"2024-09-13T14:36:33.256415+0000","last_fullsized":"2024-09-13T14:36:33.256415+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:41.568032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"264'7571","reported_seq":7616,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.973418+0000","last_change":"2024-09-13T14:34:57.526403+0000","last_active":"2024-09-13T14:36:42.973418+0000","last_peered":"2024-09-13T14:36:42.973418+0000","last_clean":"2024-09-13T14:36:42.973418+0000","last_became_active":"2024-09-13T14:34:57.525671+0000","last_became_peered":"2024-09-13T14:34:57.525671+0000","last_unstale":"2024-09-13T14:36:42.973418+0000","last_undegraded":"2024-09-13T14:36:42.973418+0000","last_fullsized":"2024-09-13T14:36:42.973418+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:22.342551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479657984,"num_objects":7319,"num_object_clones":0,"num_object_copies":21957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7571,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":212,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263760+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:36:33.263760+0000","last_peered":"2024-09-13T14:36:33.263760+0000","last_clean":"2024-09-13T14:36:33.263760+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:36:33.263760+0000","last_undegraded":"2024-09-13T14:36:33.263760+0000","last_fullsized":"2024-09-13T14:36:33.263760+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":244,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258145+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:36:33.258145+0000","last_peered":"2024-09-13T14:36:33.258145+0000","last_clean":"2024-09-13T14:36:33.258145+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:36:33.258145+0000","last_undegraded":"2024-09-13T14:36:33.258145+0000","last_fullsized":"2024-09-13T14:36:33.258145+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"265'7904","reported_seq":7951,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:46.571241+0000","last_change":"2024-09-13T14:34:57.521543+0000","last_active":"2024-09-13T14:36:46.571241+0000","last_peered":"2024-09-13T14:36:46.571241+0000","last_clean":"2024-09-13T14:36:46.571241+0000","last_became_active":"2024-09-13T14:34:57.521269+0000","last_became_peered":"2024-09-13T14:34:57.521269+0000","last_unstale":"2024-09-13T14:36:46.571241+0000","last_undegraded":"2024-09-13T14:36:46.571241+0000","last_fullsized":"2024-09-13T14:36:46.571241+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:21.067198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474873856,"num_objects":7246,"num_object_clones":0,"num_object_copies":21738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7904,"num_write_kb":484800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":229,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272098+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:36:40.272098+0000","last_peered":"2024-09-13T14:36:40.272098+0000","last_clean":"2024-09-13T14:36:40.272098+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:36:40.272098+0000","last_undegraded":"2024-09-13T14:36:40.272098+0000","last_fullsized":"2024-09-13T14:36:40.272098+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":405,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285059+0000","last_change":"2024-09-13T14:34:46.616314+0000","last_active":"2024-09-13T14:36:40.285059+0000","last_peered":"2024-09-13T14:36:40.285059+0000","last_clean":"2024-09-13T14:36:40.285059+0000","last_became_active":"2024-09-13T14:34:46.611776+0000","last_became_peered":"2024-09-13T14:34:46.611776+0000","last_unstale":"2024-09-13T14:36:40.285059+0000","last_undegraded":"2024-09-13T14:36:40.285059+0000","last_fullsized":"2024-09-13T14:36:40.285059+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:44.754447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"264'7531","reported_seq":7576,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.999957+0000","last_change":"2024-09-13T14:34:57.526751+0000","last_active":"2024-09-13T14:36:42.999957+0000","last_peered":"2024-09-13T14:36:42.999957+0000","last_clean":"2024-09-13T14:36:42.999957+0000","last_became_active":"2024-09-13T14:34:57.526442+0000","last_became_peered":"2024-09-13T14:34:57.526442+0000","last_unstale":"2024-09-13T14:36:42.999957+0000","last_undegraded":"2024-09-13T14:36:42.999957+0000","last_fullsized":"2024-09-13T14:36:42.999957+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:09.948770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475987968,"num_objects":7263,"num_object_clones":0,"num_object_copies":21789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7531,"num_write_kb":473408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":375,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263207+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:36:33.263207+0000","last_peered":"2024-09-13T14:36:33.263207+0000","last_clean":"2024-09-13T14:36:33.263207+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:36:33.263207+0000","last_undegraded":"2024-09-13T14:36:33.263207+0000","last_fullsized":"2024-09-13T14:36:33.263207+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":397,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.133497+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:36:43.133497+0000","last_peered":"2024-09-13T14:36:43.133497+0000","last_clean":"2024-09-13T14:36:43.133497+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:36:43.133497+0000","last_undegraded":"2024-09-13T14:36:43.133497+0000","last_fullsized":"2024-09-13T14:36:43.133497+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":228,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256341+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:36:33.256341+0000","last_peered":"2024-09-13T14:36:33.256341+0000","last_clean":"2024-09-13T14:36:33.256341+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:36:33.256341+0000","last_undegraded":"2024-09-13T14:36:33.256341+0000","last_fullsized":"2024-09-13T14:36:33.256341+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"265'7810","reported_seq":7857,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:46.592087+0000","last_change":"2024-09-13T14:34:57.525034+0000","last_active":"2024-09-13T14:36:46.592087+0000","last_peered":"2024-09-13T14:36:46.592087+0000","last_clean":"2024-09-13T14:36:46.592087+0000","last_became_active":"2024-09-13T14:34:57.524740+0000","last_became_peered":"2024-09-13T14:34:57.524740+0000","last_unstale":"2024-09-13T14:36:46.592087+0000","last_undegraded":"2024-09-13T14:36:46.592087+0000","last_fullsized":"2024-09-13T14:36:46.592087+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7810,"log_dups_size":0,"ondisk_log_size":7810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:44.966638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":466092032,"num_objects":7112,"num_object_clones":0,"num_object_copies":21336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7810,"num_write_kb":477504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":123,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271536+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:36:40.271536+0000","last_peered":"2024-09-13T14:36:40.271536+0000","last_clean":"2024-09-13T14:36:40.271536+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:36:40.271536+0000","last_undegraded":"2024-09-13T14:36:40.271536+0000","last_fullsized":"2024-09-13T14:36:40.271536+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258056+0000","last_change":"2024-09-13T14:34:46.575576+0000","last_active":"2024-09-13T14:36:33.258056+0000","last_peered":"2024-09-13T14:36:33.258056+0000","last_clean":"2024-09-13T14:36:33.258056+0000","last_became_active":"2024-09-13T14:34:46.575291+0000","last_became_peered":"2024-09-13T14:34:46.575291+0000","last_unstale":"2024-09-13T14:36:33.258056+0000","last_undegraded":"2024-09-13T14:36:33.258056+0000","last_fullsized":"2024-09-13T14:36:33.258056+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:30.242110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":238,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256300+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:36:33.256300+0000","last_peered":"2024-09-13T14:36:33.256300+0000","last_clean":"2024-09-13T14:36:33.256300+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:36:33.256300+0000","last_undegraded":"2024-09-13T14:36:33.256300+0000","last_fullsized":"2024-09-13T14:36:33.256300+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"264'7628","reported_seq":7654,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.955201+0000","last_change":"2024-09-13T14:34:57.522071+0000","last_active":"2024-09-13T14:36:42.955201+0000","last_peered":"2024-09-13T14:36:42.955201+0000","last_clean":"2024-09-13T14:36:42.955201+0000","last_became_active":"2024-09-13T14:34:57.521798+0000","last_became_peered":"2024-09-13T14:34:57.521798+0000","last_unstale":"2024-09-13T14:36:42.955201+0000","last_undegraded":"2024-09-13T14:36:42.955201+0000","last_fullsized":"2024-09-13T14:36:42.955201+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7628,"log_dups_size":0,"ondisk_log_size":7628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:47.723539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483196952,"num_objects":7374,"num_object_clones":0,"num_object_copies":22122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7374,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7628,"num_write_kb":480001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256488+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:36:33.256488+0000","last_peered":"2024-09-13T14:36:33.256488+0000","last_clean":"2024-09-13T14:36:33.256488+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:36:33.256488+0000","last_undegraded":"2024-09-13T14:36:33.256488+0000","last_fullsized":"2024-09-13T14:36:33.256488+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256421+0000","last_change":"2024-09-13T14:34:46.648302+0000","last_active":"2024-09-13T14:36:33.256421+0000","last_peered":"2024-09-13T14:36:33.256421+0000","last_clean":"2024-09-13T14:36:33.256421+0000","last_became_active":"2024-09-13T14:34:46.647761+0000","last_became_peered":"2024-09-13T14:34:46.647761+0000","last_unstale":"2024-09-13T14:36:33.256421+0000","last_undegraded":"2024-09-13T14:36:33.256421+0000","last_fullsized":"2024-09-13T14:36:33.256421+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:22.055371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"264'7338","reported_seq":7363,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.906108+0000","last_change":"2024-09-13T14:34:57.521324+0000","last_active":"2024-09-13T14:36:42.906108+0000","last_peered":"2024-09-13T14:36:42.906108+0000","last_clean":"2024-09-13T14:36:42.906108+0000","last_became_active":"2024-09-13T14:34:57.520992+0000","last_became_peered":"2024-09-13T14:34:57.520992+0000","last_unstale":"2024-09-13T14:36:42.906108+0000","last_undegraded":"2024-09-13T14:36:42.906108+0000","last_fullsized":"2024-09-13T14:36:42.906108+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.105872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":464912384,"num_objects":7094,"num_object_clones":0,"num_object_copies":21282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7338,"num_write_kb":461824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":116,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271663+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:36:40.271663+0000","last_peered":"2024-09-13T14:36:40.271663+0000","last_clean":"2024-09-13T14:36:40.271663+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:36:40.271663+0000","last_undegraded":"2024-09-13T14:36:40.271663+0000","last_fullsized":"2024-09-13T14:36:40.271663+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":411,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.130688+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:36:43.130688+0000","last_peered":"2024-09-13T14:36:43.130688+0000","last_clean":"2024-09-13T14:36:43.130688+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:36:43.130688+0000","last_undegraded":"2024-09-13T14:36:43.130688+0000","last_fullsized":"2024-09-13T14:36:43.130688+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":359,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.120494+0000","last_change":"2024-09-13T14:34:46.624474+0000","last_active":"2024-09-13T14:36:43.120494+0000","last_peered":"2024-09-13T14:36:43.120494+0000","last_clean":"2024-09-13T14:36:43.120494+0000","last_became_active":"2024-09-13T14:34:46.624138+0000","last_became_peered":"2024-09-13T14:34:46.624138+0000","last_unstale":"2024-09-13T14:36:43.120494+0000","last_undegraded":"2024-09-13T14:36:43.120494+0000","last_fullsized":"2024-09-13T14:36:43.120494+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:51.252508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"264'7521","reported_seq":7566,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.996162+0000","last_change":"2024-09-13T14:34:57.525058+0000","last_active":"2024-09-13T14:36:42.996162+0000","last_peered":"2024-09-13T14:36:42.996162+0000","last_clean":"2024-09-13T14:36:42.996162+0000","last_became_active":"2024-09-13T14:34:57.524514+0000","last_became_peered":"2024-09-13T14:34:57.524514+0000","last_unstale":"2024-09-13T14:36:42.996162+0000","last_undegraded":"2024-09-13T14:36:42.996162+0000","last_fullsized":"2024-09-13T14:36:42.996162+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:17.836817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476381184,"num_objects":7269,"num_object_clones":0,"num_object_copies":21807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7521,"num_write_kb":473280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":366,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263279+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:36:33.263279+0000","last_peered":"2024-09-13T14:36:33.263279+0000","last_clean":"2024-09-13T14:36:33.263279+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:36:33.263279+0000","last_undegraded":"2024-09-13T14:36:33.263279+0000","last_fullsized":"2024-09-13T14:36:33.263279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":830,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272232+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:36:40.272232+0000","last_peered":"2024-09-13T14:36:40.272232+0000","last_clean":"2024-09-13T14:36:40.272232+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:36:40.272232+0000","last_undegraded":"2024-09-13T14:36:40.272232+0000","last_fullsized":"2024-09-13T14:36:40.272232+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":354,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.131261+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:36:43.131261+0000","last_peered":"2024-09-13T14:36:43.131261+0000","last_clean":"2024-09-13T14:36:43.131261+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:36:43.131261+0000","last_undegraded":"2024-09-13T14:36:43.131261+0000","last_fullsized":"2024-09-13T14:36:43.131261+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":210,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258509+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:36:33.258509+0000","last_peered":"2024-09-13T14:36:33.258509+0000","last_clean":"2024-09-13T14:36:33.258509+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:36:33.258509+0000","last_undegraded":"2024-09-13T14:36:33.258509+0000","last_fullsized":"2024-09-13T14:36:33.258509+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"264'7606","reported_seq":7629,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.025677+0000","last_change":"2024-09-13T14:34:57.525197+0000","last_active":"2024-09-13T14:36:43.025677+0000","last_peered":"2024-09-13T14:36:43.025677+0000","last_clean":"2024-09-13T14:36:43.025677+0000","last_became_active":"2024-09-13T14:34:57.524916+0000","last_became_peered":"2024-09-13T14:34:57.524916+0000","last_unstale":"2024-09-13T14:36:43.025677+0000","last_undegraded":"2024-09-13T14:36:43.025677+0000","last_fullsized":"2024-09-13T14:36:43.025677+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7606,"log_dups_size":0,"ondisk_log_size":7606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:01.282676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":481951744,"num_objects":7354,"num_object_clones":0,"num_object_copies":22062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7606,"num_write_kb":478720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":198,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.121187+0000","last_change":"2024-09-13T14:34:47.477679+0000","last_active":"2024-09-13T14:36:43.121187+0000","last_peered":"2024-09-13T14:36:43.121187+0000","last_clean":"2024-09-13T14:36:43.121187+0000","last_became_active":"2024-09-13T14:34:46.634361+0000","last_became_peered":"2024-09-13T14:34:46.634361+0000","last_unstale":"2024-09-13T14:36:43.121187+0000","last_undegraded":"2024-09-13T14:36:43.121187+0000","last_fullsized":"2024-09-13T14:36:43.121187+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:44.197390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":654,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263276+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:36:33.263276+0000","last_peered":"2024-09-13T14:36:33.263276+0000","last_clean":"2024-09-13T14:36:33.263276+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:36:33.263276+0000","last_undegraded":"2024-09-13T14:36:33.263276+0000","last_fullsized":"2024-09-13T14:36:33.263276+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":394,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282187+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:36:40.282187+0000","last_peered":"2024-09-13T14:36:40.282187+0000","last_clean":"2024-09-13T14:36:40.282187+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:36:40.282187+0000","last_undegraded":"2024-09-13T14:36:40.282187+0000","last_fullsized":"2024-09-13T14:36:40.282187+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":416,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282777+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:36:40.282777+0000","last_peered":"2024-09-13T14:36:40.282777+0000","last_clean":"2024-09-13T14:36:40.282777+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:36:40.282777+0000","last_undegraded":"2024-09-13T14:36:40.282777+0000","last_fullsized":"2024-09-13T14:36:40.282777+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"264'7580","reported_seq":7625,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.999836+0000","last_change":"2024-09-13T14:34:57.525939+0000","last_active":"2024-09-13T14:36:42.999836+0000","last_peered":"2024-09-13T14:36:42.999836+0000","last_clean":"2024-09-13T14:36:42.999836+0000","last_became_active":"2024-09-13T14:34:57.525621+0000","last_became_peered":"2024-09-13T14:34:57.525621+0000","last_unstale":"2024-09-13T14:36:42.999836+0000","last_undegraded":"2024-09-13T14:36:42.999836+0000","last_fullsized":"2024-09-13T14:36:42.999836+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:26.471406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480378880,"num_objects":7330,"num_object_clones":0,"num_object_copies":21990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7580,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":375,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263371+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:36:33.263371+0000","last_peered":"2024-09-13T14:36:33.263371+0000","last_clean":"2024-09-13T14:36:33.263371+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:36:33.263371+0000","last_undegraded":"2024-09-13T14:36:33.263371+0000","last_fullsized":"2024-09-13T14:36:33.263371+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":825,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.132336+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:36:43.132336+0000","last_peered":"2024-09-13T14:36:43.132336+0000","last_clean":"2024-09-13T14:36:43.132336+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:36:43.132336+0000","last_undegraded":"2024-09-13T14:36:43.132336+0000","last_fullsized":"2024-09-13T14:36:43.132336+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":351,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.123210+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:36:43.123210+0000","last_peered":"2024-09-13T14:36:43.123210+0000","last_clean":"2024-09-13T14:36:43.123210+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:36:43.123210+0000","last_undegraded":"2024-09-13T14:36:43.123210+0000","last_fullsized":"2024-09-13T14:36:43.123210+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":402,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.132725+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:36:43.132725+0000","last_peered":"2024-09-13T14:36:43.132725+0000","last_clean":"2024-09-13T14:36:43.132725+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:36:43.132725+0000","last_undegraded":"2024-09-13T14:36:43.132725+0000","last_fullsized":"2024-09-13T14:36:43.132725+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"264'7660","reported_seq":7685,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:42.949593+0000","last_change":"2024-09-13T14:34:57.522448+0000","last_active":"2024-09-13T14:36:42.949593+0000","last_peered":"2024-09-13T14:36:42.949593+0000","last_clean":"2024-09-13T14:36:42.949593+0000","last_became_active":"2024-09-13T14:34:57.522167+0000","last_became_peered":"2024-09-13T14:34:57.522167+0000","last_unstale":"2024-09-13T14:36:42.949593+0000","last_undegraded":"2024-09-13T14:36:42.949593+0000","last_fullsized":"2024-09-13T14:36:42.949593+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7660,"log_dups_size":0,"ondisk_log_size":7660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:22:47.802147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":484442112,"num_objects":7392,"num_object_clones":0,"num_object_copies":22176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7660,"num_write_kb":481664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":116,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271534+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:36:40.271534+0000","last_peered":"2024-09-13T14:36:40.271534+0000","last_clean":"2024-09-13T14:36:40.271534+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:36:40.271534+0000","last_undegraded":"2024-09-13T14:36:40.271534+0000","last_fullsized":"2024-09-13T14:36:40.271534+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":896,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271829+0000","last_change":"2024-09-13T14:34:46.566197+0000","last_active":"2024-09-13T14:36:40.271829+0000","last_peered":"2024-09-13T14:36:40.271829+0000","last_clean":"2024-09-13T14:36:40.271829+0000","last_became_active":"2024-09-13T14:34:46.565768+0000","last_became_peered":"2024-09-13T14:34:46.565768+0000","last_unstale":"2024-09-13T14:36:40.271829+0000","last_undegraded":"2024-09-13T14:36:40.271829+0000","last_fullsized":"2024-09-13T14:36:40.271829+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:12:46.443713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":229,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256687+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:36:33.256687+0000","last_peered":"2024-09-13T14:36:33.256687+0000","last_clean":"2024-09-13T14:36:33.256687+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:36:33.256687+0000","last_undegraded":"2024-09-13T14:36:33.256687+0000","last_fullsized":"2024-09-13T14:36:33.256687+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":371,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.121338+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:36:43.121338+0000","last_peered":"2024-09-13T14:36:43.121338+0000","last_clean":"2024-09-13T14:36:43.121338+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:36:43.121338+0000","last_undegraded":"2024-09-13T14:36:43.121338+0000","last_fullsized":"2024-09-13T14:36:43.121338+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"264'7583","reported_seq":7606,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.089961+0000","last_change":"2024-09-13T14:34:57.521524+0000","last_active":"2024-09-13T14:36:43.089961+0000","last_peered":"2024-09-13T14:36:43.089961+0000","last_clean":"2024-09-13T14:36:43.089961+0000","last_became_active":"2024-09-13T14:34:57.520850+0000","last_became_peered":"2024-09-13T14:34:57.520850+0000","last_unstale":"2024-09-13T14:36:43.089961+0000","last_undegraded":"2024-09-13T14:36:43.089961+0000","last_fullsized":"2024-09-13T14:36:43.089961+0000","mapping_epoch":247,"log_start":"251'200","ondisk_log_start":"251'200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:42.515897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477954048,"num_objects":7293,"num_object_clones":0,"num_object_copies":21879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7583,"num_write_kb":476032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":263,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258427+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:36:33.258427+0000","last_peered":"2024-09-13T14:36:33.258427+0000","last_clean":"2024-09-13T14:36:33.258427+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:36:33.258427+0000","last_undegraded":"2024-09-13T14:36:33.258427+0000","last_fullsized":"2024-09-13T14:36:33.258427+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272410+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:36:40.272410+0000","last_peered":"2024-09-13T14:36:40.272410+0000","last_clean":"2024-09-13T14:36:40.272410+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:36:40.272410+0000","last_undegraded":"2024-09-13T14:36:40.272410+0000","last_fullsized":"2024-09-13T14:36:40.272410+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":950,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.121499+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:36:43.121499+0000","last_peered":"2024-09-13T14:36:43.121499+0000","last_clean":"2024-09-13T14:36:43.121499+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:36:43.121499+0000","last_undegraded":"2024-09-13T14:36:43.121499+0000","last_fullsized":"2024-09-13T14:36:43.121499+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":329,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.120817+0000","last_change":"2024-09-13T14:34:46.616754+0000","last_active":"2024-09-13T14:36:43.120817+0000","last_peered":"2024-09-13T14:36:43.120817+0000","last_clean":"2024-09-13T14:36:43.120817+0000","last_became_active":"2024-09-13T14:34:46.616476+0000","last_became_peered":"2024-09-13T14:34:46.616476+0000","last_unstale":"2024-09-13T14:36:43.120817+0000","last_undegraded":"2024-09-13T14:36:43.120817+0000","last_fullsized":"2024-09-13T14:36:43.120817+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:14.670350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"264'7594","reported_seq":7620,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.060856+0000","last_change":"2024-09-13T14:34:57.527395+0000","last_active":"2024-09-13T14:36:43.060856+0000","last_peered":"2024-09-13T14:36:43.060856+0000","last_clean":"2024-09-13T14:36:43.060856+0000","last_became_active":"2024-09-13T14:34:57.527133+0000","last_became_peered":"2024-09-13T14:34:57.527133+0000","last_unstale":"2024-09-13T14:36:43.060856+0000","last_undegraded":"2024-09-13T14:36:43.060856+0000","last_fullsized":"2024-09-13T14:36:43.060856+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:16.255103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479461376,"num_objects":7316,"num_object_clones":0,"num_object_copies":21948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7594,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":263,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258327+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:36:33.258327+0000","last_peered":"2024-09-13T14:36:33.258327+0000","last_clean":"2024-09-13T14:36:33.258327+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:36:33.258327+0000","last_undegraded":"2024-09-13T14:36:33.258327+0000","last_fullsized":"2024-09-13T14:36:33.258327+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":350,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.121418+0000","last_change":"2024-09-13T14:34:46.583932+0000","last_active":"2024-09-13T14:36:43.121418+0000","last_peered":"2024-09-13T14:36:43.121418+0000","last_clean":"2024-09-13T14:36:43.121418+0000","last_became_active":"2024-09-13T14:34:46.580751+0000","last_became_peered":"2024-09-13T14:34:46.580751+0000","last_unstale":"2024-09-13T14:36:43.121418+0000","last_undegraded":"2024-09-13T14:36:43.121418+0000","last_fullsized":"2024-09-13T14:36:43.121418+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:38.905892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":574,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263773+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:36:33.263773+0000","last_peered":"2024-09-13T14:36:33.263773+0000","last_clean":"2024-09-13T14:36:33.263773+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:36:33.263773+0000","last_undegraded":"2024-09-13T14:36:33.263773+0000","last_fullsized":"2024-09-13T14:36:33.263773+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":384,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.284967+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:36:40.284967+0000","last_peered":"2024-09-13T14:36:40.284967+0000","last_clean":"2024-09-13T14:36:40.284967+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:36:40.284967+0000","last_undegraded":"2024-09-13T14:36:40.284967+0000","last_fullsized":"2024-09-13T14:36:40.284967+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"265'7558","reported_seq":7604,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:44.852543+0000","last_change":"2024-09-13T14:34:57.521462+0000","last_active":"2024-09-13T14:36:44.852543+0000","last_peered":"2024-09-13T14:36:44.852543+0000","last_clean":"2024-09-13T14:36:44.852543+0000","last_became_active":"2024-09-13T14:34:57.521178+0000","last_became_peered":"2024-09-13T14:34:57.521178+0000","last_unstale":"2024-09-13T14:36:44.852543+0000","last_undegraded":"2024-09-13T14:36:44.852543+0000","last_fullsized":"2024-09-13T14:36:44.852543+0000","mapping_epoch":247,"log_start":"252'800","ondisk_log_start":"252'800","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:32.574385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":466485248,"num_objects":7118,"num_object_clones":0,"num_object_copies":21354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7558,"num_write_kb":469632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":115,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256193+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:36:33.256193+0000","last_peered":"2024-09-13T14:36:33.256193+0000","last_clean":"2024-09-13T14:36:33.256193+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:36:33.256193+0000","last_undegraded":"2024-09-13T14:36:33.256193+0000","last_fullsized":"2024-09-13T14:36:33.256193+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":397,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.131682+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:36:43.131682+0000","last_peered":"2024-09-13T14:36:43.131682+0000","last_clean":"2024-09-13T14:36:43.131682+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:36:43.131682+0000","last_undegraded":"2024-09-13T14:36:43.131682+0000","last_fullsized":"2024-09-13T14:36:43.131682+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":684,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.120249+0000","last_change":"2024-09-13T14:34:46.652006+0000","last_active":"2024-09-13T14:36:43.120249+0000","last_peered":"2024-09-13T14:36:43.120249+0000","last_clean":"2024-09-13T14:36:43.120249+0000","last_became_active":"2024-09-13T14:34:46.651532+0000","last_became_peered":"2024-09-13T14:34:46.651532+0000","last_unstale":"2024-09-13T14:36:43.120249+0000","last_undegraded":"2024-09-13T14:36:43.120249+0000","last_fullsized":"2024-09-13T14:36:43.120249+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:35.907330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":395,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.130501+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:36:43.130501+0000","last_peered":"2024-09-13T14:36:43.130501+0000","last_clean":"2024-09-13T14:36:43.130501+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:36:43.130501+0000","last_undegraded":"2024-09-13T14:36:43.130501+0000","last_fullsized":"2024-09-13T14:36:43.130501+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"265'7692","reported_seq":7739,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:44.884762+0000","last_change":"2024-09-13T14:34:57.522594+0000","last_active":"2024-09-13T14:36:44.884762+0000","last_peered":"2024-09-13T14:36:44.884762+0000","last_clean":"2024-09-13T14:36:44.884762+0000","last_became_active":"2024-09-13T14:34:57.520039+0000","last_became_peered":"2024-09-13T14:34:57.520039+0000","last_unstale":"2024-09-13T14:36:44.884762+0000","last_undegraded":"2024-09-13T14:36:44.884762+0000","last_fullsized":"2024-09-13T14:36:44.884762+0000","mapping_epoch":247,"log_start":"252'1000","ondisk_log_start":"252'1000","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:37.564876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474611712,"num_objects":7242,"num_object_clones":0,"num_object_copies":21726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7692,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":236,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.134313+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:36:43.134313+0000","last_peered":"2024-09-13T14:36:43.134313+0000","last_clean":"2024-09-13T14:36:43.134313+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:36:43.134313+0000","last_undegraded":"2024-09-13T14:36:43.134313+0000","last_fullsized":"2024-09-13T14:36:43.134313+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":408,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282551+0000","last_change":"2024-09-13T14:34:46.577789+0000","last_active":"2024-09-13T14:36:40.282551+0000","last_peered":"2024-09-13T14:36:40.282551+0000","last_clean":"2024-09-13T14:36:40.282551+0000","last_became_active":"2024-09-13T14:34:46.574952+0000","last_became_peered":"2024-09-13T14:34:46.574952+0000","last_unstale":"2024-09-13T14:36:40.282551+0000","last_undegraded":"2024-09-13T14:36:40.282551+0000","last_fullsized":"2024-09-13T14:36:40.282551+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:20:04.720034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":682,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258515+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:36:33.258515+0000","last_peered":"2024-09-13T14:36:33.258515+0000","last_clean":"2024-09-13T14:36:33.258515+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:36:33.258515+0000","last_undegraded":"2024-09-13T14:36:33.258515+0000","last_fullsized":"2024-09-13T14:36:33.258515+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":211,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256184+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:36:33.256184+0000","last_peered":"2024-09-13T14:36:33.256184+0000","last_clean":"2024-09-13T14:36:33.256184+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:36:33.256184+0000","last_undegraded":"2024-09-13T14:36:33.256184+0000","last_fullsized":"2024-09-13T14:36:33.256184+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256583+0000","last_change":"2024-09-13T14:34:46.645586+0000","last_active":"2024-09-13T14:36:33.256583+0000","last_peered":"2024-09-13T14:36:33.256583+0000","last_clean":"2024-09-13T14:36:33.256583+0000","last_became_active":"2024-09-13T14:34:46.645294+0000","last_became_peered":"2024-09-13T14:34:46.645294+0000","last_unstale":"2024-09-13T14:36:33.256583+0000","last_undegraded":"2024-09-13T14:36:33.256583+0000","last_fullsized":"2024-09-13T14:36:33.256583+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:55.643942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":234,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256582+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:36:33.256582+0000","last_peered":"2024-09-13T14:36:33.256582+0000","last_clean":"2024-09-13T14:36:33.256582+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:36:33.256582+0000","last_undegraded":"2024-09-13T14:36:33.256582+0000","last_fullsized":"2024-09-13T14:36:33.256582+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":349,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.120696+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:36:43.120696+0000","last_peered":"2024-09-13T14:36:43.120696+0000","last_clean":"2024-09-13T14:36:43.120696+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:36:43.120696+0000","last_undegraded":"2024-09-13T14:36:43.120696+0000","last_fullsized":"2024-09-13T14:36:43.120696+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":231,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.263690+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:36:33.263690+0000","last_peered":"2024-09-13T14:36:33.263690+0000","last_clean":"2024-09-13T14:36:33.263690+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:36:33.263690+0000","last_undegraded":"2024-09-13T14:36:33.263690+0000","last_fullsized":"2024-09-13T14:36:33.263690+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272515+0000","last_change":"2024-09-13T14:34:46.565021+0000","last_active":"2024-09-13T14:36:40.272515+0000","last_peered":"2024-09-13T14:36:40.272515+0000","last_clean":"2024-09-13T14:36:40.272515+0000","last_became_active":"2024-09-13T14:34:46.563784+0000","last_became_peered":"2024-09-13T14:34:46.563784+0000","last_unstale":"2024-09-13T14:36:40.272515+0000","last_undegraded":"2024-09-13T14:36:40.272515+0000","last_fullsized":"2024-09-13T14:36:40.272515+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:00.380107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":354,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.130956+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:36:43.130956+0000","last_peered":"2024-09-13T14:36:43.130956+0000","last_clean":"2024-09-13T14:36:43.130956+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:36:43.130956+0000","last_undegraded":"2024-09-13T14:36:43.130956+0000","last_fullsized":"2024-09-13T14:36:43.130956+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":393,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.294683+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:36:40.294683+0000","last_peered":"2024-09-13T14:36:40.294683+0000","last_clean":"2024-09-13T14:36:40.294683+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:36:40.294683+0000","last_undegraded":"2024-09-13T14:36:40.294683+0000","last_fullsized":"2024-09-13T14:36:40.294683+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":410,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.130420+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:36:43.130420+0000","last_peered":"2024-09-13T14:36:43.130420+0000","last_clean":"2024-09-13T14:36:43.130420+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:36:43.130420+0000","last_undegraded":"2024-09-13T14:36:43.130420+0000","last_fullsized":"2024-09-13T14:36:43.130420+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":244,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.258484+0000","last_change":"2024-09-13T14:34:46.570197+0000","last_active":"2024-09-13T14:36:33.258484+0000","last_peered":"2024-09-13T14:36:33.258484+0000","last_clean":"2024-09-13T14:36:33.258484+0000","last_became_active":"2024-09-13T14:34:46.568152+0000","last_became_peered":"2024-09-13T14:34:46.568152+0000","last_unstale":"2024-09-13T14:36:33.258484+0000","last_undegraded":"2024-09-13T14:36:33.258484+0000","last_fullsized":"2024-09-13T14:36:33.258484+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:24:44.592857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":428,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.130586+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:36:43.130586+0000","last_peered":"2024-09-13T14:36:43.130586+0000","last_clean":"2024-09-13T14:36:43.130586+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:36:43.130586+0000","last_undegraded":"2024-09-13T14:36:43.130586+0000","last_fullsized":"2024-09-13T14:36:43.130586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":395,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.282733+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:36:40.282733+0000","last_peered":"2024-09-13T14:36:40.282733+0000","last_clean":"2024-09-13T14:36:40.282733+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:36:40.282733+0000","last_undegraded":"2024-09-13T14:36:40.282733+0000","last_fullsized":"2024-09-13T14:36:40.282733+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":360,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.121127+0000","last_change":"2024-09-13T14:34:46.624853+0000","last_active":"2024-09-13T14:36:43.121127+0000","last_peered":"2024-09-13T14:36:43.121127+0000","last_clean":"2024-09-13T14:36:43.121127+0000","last_became_active":"2024-09-13T14:34:46.624553+0000","last_became_peered":"2024-09-13T14:34:46.624553+0000","last_unstale":"2024-09-13T14:36:43.121127+0000","last_undegraded":"2024-09-13T14:36:43.121127+0000","last_fullsized":"2024-09-13T14:36:43.121127+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:22.679066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":410,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.133600+0000","last_change":"2024-09-13T14:34:46.567930+0000","last_active":"2024-09-13T14:36:43.133600+0000","last_peered":"2024-09-13T14:36:43.133600+0000","last_clean":"2024-09-13T14:36:43.133600+0000","last_became_active":"2024-09-13T14:34:46.567500+0000","last_became_peered":"2024-09-13T14:34:46.567500+0000","last_unstale":"2024-09-13T14:36:43.133600+0000","last_undegraded":"2024-09-13T14:36:43.133600+0000","last_fullsized":"2024-09-13T14:36:43.133600+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:47.997991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":264,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256874+0000","last_change":"2024-09-13T14:34:46.581764+0000","last_active":"2024-09-13T14:36:33.256874+0000","last_peered":"2024-09-13T14:36:33.256874+0000","last_clean":"2024-09-13T14:36:33.256874+0000","last_became_active":"2024-09-13T14:34:46.581377+0000","last_became_peered":"2024-09-13T14:34:46.581377+0000","last_unstale":"2024-09-13T14:36:33.256874+0000","last_undegraded":"2024-09-13T14:36:33.256874+0000","last_fullsized":"2024-09-13T14:36:33.256874+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:28.920200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":430,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.132578+0000","last_change":"2024-09-13T14:34:46.582553+0000","last_active":"2024-09-13T14:36:43.132578+0000","last_peered":"2024-09-13T14:36:43.132578+0000","last_clean":"2024-09-13T14:36:43.132578+0000","last_became_active":"2024-09-13T14:34:46.582146+0000","last_became_peered":"2024-09-13T14:34:46.582146+0000","last_unstale":"2024-09-13T14:36:43.132578+0000","last_undegraded":"2024-09-13T14:36:43.132578+0000","last_fullsized":"2024-09-13T14:36:43.132578+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:55:13.365210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":386,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.284850+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:36:40.284850+0000","last_peered":"2024-09-13T14:36:40.284850+0000","last_clean":"2024-09-13T14:36:40.284850+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:36:40.284850+0000","last_undegraded":"2024-09-13T14:36:40.284850+0000","last_fullsized":"2024-09-13T14:36:40.284850+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":262,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256610+0000","last_change":"2024-09-13T14:34:46.584115+0000","last_active":"2024-09-13T14:36:33.256610+0000","last_peered":"2024-09-13T14:36:33.256610+0000","last_clean":"2024-09-13T14:36:33.256610+0000","last_became_active":"2024-09-13T14:34:46.583779+0000","last_became_peered":"2024-09-13T14:34:46.583779+0000","last_unstale":"2024-09-13T14:36:33.256610+0000","last_undegraded":"2024-09-13T14:36:33.256610+0000","last_fullsized":"2024-09-13T14:36:33.256610+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:19.009728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272024+0000","last_change":"2024-09-13T14:34:46.663283+0000","last_active":"2024-09-13T14:36:40.272024+0000","last_peered":"2024-09-13T14:36:40.272024+0000","last_clean":"2024-09-13T14:36:40.272024+0000","last_became_active":"2024-09-13T14:34:46.662866+0000","last_became_peered":"2024-09-13T14:34:46.662866+0000","last_unstale":"2024-09-13T14:36:40.272024+0000","last_undegraded":"2024-09-13T14:36:40.272024+0000","last_fullsized":"2024-09-13T14:36:40.272024+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:17:19.510340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.256775+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:36:33.256775+0000","last_peered":"2024-09-13T14:36:33.256775+0000","last_clean":"2024-09-13T14:36:33.256775+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:36:33.256775+0000","last_undegraded":"2024-09-13T14:36:33.256775+0000","last_fullsized":"2024-09-13T14:36:33.256775+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":214,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272381+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:36:40.272381+0000","last_peered":"2024-09-13T14:36:40.272381+0000","last_clean":"2024-09-13T14:36:40.272381+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:36:40.272381+0000","last_undegraded":"2024-09-13T14:36:40.272381+0000","last_fullsized":"2024-09-13T14:36:40.272381+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":420,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.131254+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:36:43.131254+0000","last_peered":"2024-09-13T14:36:43.131254+0000","last_clean":"2024-09-13T14:36:43.131254+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:36:43.131254+0000","last_undegraded":"2024-09-13T14:36:43.131254+0000","last_fullsized":"2024-09-13T14:36:43.131254+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":406,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.133039+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:36:43.133039+0000","last_peered":"2024-09-13T14:36:43.133039+0000","last_clean":"2024-09-13T14:36:43.133039+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:36:43.133039+0000","last_undegraded":"2024-09-13T14:36:43.133039+0000","last_fullsized":"2024-09-13T14:36:43.133039+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":238,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-09-13T14:36:33.255909+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:36:33.255909+0000","last_peered":"2024-09-13T14:36:33.255909+0000","last_clean":"2024-09-13T14:36:33.255909+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:36:33.255909+0000","last_undegraded":"2024-09-13T14:36:33.255909+0000","last_fullsized":"2024-09-13T14:36:33.255909+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":398,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.294233+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:36:40.294233+0000","last_peered":"2024-09-13T14:36:40.294233+0000","last_clean":"2024-09-13T14:36:40.294233+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:36:40.294233+0000","last_undegraded":"2024-09-13T14:36:40.294233+0000","last_fullsized":"2024-09-13T14:36:40.294233+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":7619543064,"num_objects":116266,"num_object_clones":0,"num_object_copies":348798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116266,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121998,"num_write_kb":7624385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22841733120,"data_stored":22841720904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118998,"ondisk_log_size":118998,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397807,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2647624,"kb_used_data":2346112,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91097528,"statfs":{"total":95995035648,"available":93283868672,"internally_reserved":0,"allocated":2402418688,"data_stored":2401106836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594035,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649532,"kb_used_data":2348020,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91095620,"statfs":{"total":95995035648,"available":93281914880,"internally_reserved":0,"allocated":2404372480,"data_stored":2403072246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790265,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184372,"kb_used_data":1887020,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91560780,"statfs":{"total":95995035648,"available":93758238720,"internally_reserved":0,"allocated":1932308480,"data_stored":1931012721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986493,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2658820,"kb_used_data":2357308,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91086332,"statfs":{"total":95995035648,"available":93272403968,"internally_reserved":0,"allocated":2413883392,"data_stored":2412575566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697051,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4727296,"kb_used_data":4175608,"kb_used_omap":4,"kb_used_meta":551675,"kb_avail":89017856,"statfs":{"total":95995035648,"available":91154284544,"internally_reserved":0,"allocated":4275822592,"data_stored":4274522952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":564915275},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860578,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3192388,"kb_used_data":2755708,"kb_used_omap":8,"kb_used_meta":436599,"kb_avail":90552764,"statfs":{"total":95995035648,"available":92726030336,"internally_reserved":0,"allocated":2821844992,"data_stored":2820541778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":447078220},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020249,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2905348,"kb_used_data":2793592,"kb_used_omap":9,"kb_used_meta":111734,"kb_avail":90839804,"statfs":{"total":95995035648,"available":93019959296,"internally_reserved":0,"allocated":2860638208,"data_stored":2859337977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":114416453},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187639,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4183416,"kb_used_data":3766572,"kb_used_omap":7,"kb_used_meta":416824,"kb_avail":89561736,"statfs":{"total":95995035648,"available":91711217664,"internally_reserved":0,"allocated":3856969728,"data_stored":3855673717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":426828107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3841724416,"data_stored":3841720344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2848460800,"data_stored":2848456728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2812936192,"data_stored":2812936192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4237950976,"data_stored":4237950976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2399731712,"data_stored":2399731712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1923809280,"data_stored":1923809280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2389970944,"data_stored":2389966872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2387148800,"data_stored":2387148800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:36:54.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:36:54.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:54.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:54.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:54.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:54.419 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:54.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:54.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:54.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:54.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:54 smithi017 ceph-mon[107556]: pgmap v747: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.8 MiB/s wr, 752 op/s 2024-09-13T14:36:54.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:54 smithi017 ceph-mon[103001]: pgmap v747: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.8 MiB/s wr, 752 op/s 2024-09-13T14:36:54.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:36:54.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:54 smithi189 ceph-mon[90992]: pgmap v747: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 3.8 MiB/s wr, 752 op/s 2024-09-13T14:36:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:54.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:54.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:54.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:54.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:55.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:55.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:55.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:55.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:55.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:55.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:55.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:55.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:55.538 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:55.621 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:55.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:55.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:55.786 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:55.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:55.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:55.943 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:55.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:56.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:56.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:56.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:56.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:56.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:56.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:56.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:56.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:56.595 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:56 smithi017 ceph-mon[103001]: pgmap v748: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 1.6 MiB/s wr, 832 op/s 2024-09-13T14:36:56.595 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:56 smithi017 ceph-mon[107556]: pgmap v748: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 1.6 MiB/s wr, 832 op/s 2024-09-13T14:36:56.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:56.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:56 smithi189 ceph-mon[90992]: pgmap v748: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 1.6 MiB/s wr, 832 op/s 2024-09-13T14:36:56.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:56.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:56.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:36:56.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:56.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:36:57.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:57.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:57.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:57.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:57.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:57.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:57.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:36:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:36:57] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:36:57.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:57.482 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:57.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:57.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:57.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:57.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:57.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:57.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:57.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:57.896 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:57.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:57.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:58.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:36:58.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:36:58.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:58.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: pgmap v749: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3938163227' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.246 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: pgmap v749: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3938163227' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.303 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: pgmap v749: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3938163227' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:36:58.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:58.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:58.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:58.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:58.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:58.578 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:58.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:58.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:58.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:58.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:58.874 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:58.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:36:58.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:36:59.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:36:59.074 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:36:59.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:59.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[103001]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:36:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:36:59 smithi017 ceph-mon[107556]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:36:59.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:36:59.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:36:59.294 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:59 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:36:59.294 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:59 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:36:59.294 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:36:59 smithi189 ceph-mon[90992]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:36:59.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:59.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:59.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:36:59.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:59.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:36:59.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:36:59.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:36:59.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:36:59.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:36:59.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:36:59.959 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:36:59.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:36:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:36:59.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:00.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:00.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:00.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:00.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:00.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:00.364 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:00.367 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:00.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:00.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:00.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:00.616 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:00.646 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:00 smithi017 ceph-mon[103001]: pgmap v751: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-09-13T14:37:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:00 smithi017 ceph-mon[107556]: pgmap v751: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-09-13T14:37:00.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:00.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:00 smithi189 ceph-mon[90992]: pgmap v751: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-09-13T14:37:00.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:00.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:00.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:00.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:01.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:01.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:01.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:01.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:01.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:01.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:01.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:01.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:01.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:01.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:01.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:01.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:01.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:01.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:01.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:01.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:01.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:02.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:02.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:02.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:02.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:02.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:02.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:02.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:02.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:02.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:02 smithi017 ceph-mon[103001]: pgmap v752: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T14:37:02.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:02 smithi017 ceph-mon[107556]: pgmap v752: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T14:37:02.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:02.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:02.703 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:37:02.703 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:37:02.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:02.751 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:37:02.752 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:37:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:02 smithi189 ceph-mon[90992]: pgmap v752: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-09-13T14:37:02.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:02.794 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:37:02.794 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:37:02.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:37:02.831 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:37:02.866 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:37:02.866 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:37:02.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:02.901 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:37:02.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:37:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:02.926 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:02.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:02.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:37:02.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:37:02.963 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:37:02.964 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:37:03.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:03.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:03.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:03.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:03.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:03.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:03.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:03.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:03.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2592692677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2592692677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:03.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:03 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2592692677' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:03.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:03.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:37:03.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:03.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:03.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:03.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:03.986 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:37:03.990 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:37:04.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:04.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:04.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:04.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:04.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:04.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:04.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:04.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:04.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:04.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:04.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:04.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:04.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:04.748 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:04 smithi189 ceph-mon[90992]: pgmap v753: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:37:04.850 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:04.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:04.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:04 smithi017 ceph-mon[103001]: pgmap v753: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:37:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:04 smithi017 ceph-mon[107556]: pgmap v753: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:37:04.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:05.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:05.043 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":753,"stamp":"2024-09-13T14:37:03.057060+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6770792421,"num_objects":103270,"num_object_clones":0,"num_object_copies":309810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103270,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":135929,"num_write_kb":7637008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127954,"ondisk_log_size":127954,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22779108,"kb_used_data":19973472,"kb_used_omap":36,"kb_used_meta":2805403,"kb_avail":727182108,"statfs":{"total":767960285184,"available":744634478592,"internally_reserved":0,"allocated":20452835328,"data_stored":20442384975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37497,"internal_metadata":2872733063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,3,8,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-521797632,"num_objects":-7962,"num_object_clones":0,"num_object_copies":-23886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7962,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003411"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":412,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.384051+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:36:58.384051+0000","last_peered":"2024-09-13T14:36:58.384051+0000","last_clean":"2024-09-13T14:36:58.384051+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:36:58.384051+0000","last_undegraded":"2024-09-13T14:36:58.384051+0000","last_fullsized":"2024-09-13T14:36:58.384051+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":231,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271950+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:36:40.271950+0000","last_peered":"2024-09-13T14:36:40.271950+0000","last_clean":"2024-09-13T14:36:40.271950+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:36:40.271950+0000","last_undegraded":"2024-09-13T14:36:40.271950+0000","last_fullsized":"2024-09-13T14:36:40.271950+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272299+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:36:40.272299+0000","last_peered":"2024-09-13T14:36:40.272299+0000","last_clean":"2024-09-13T14:36:40.272299+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:36:40.272299+0000","last_undegraded":"2024-09-13T14:36:40.272299+0000","last_fullsized":"2024-09-13T14:36:40.272299+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":395,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125759+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:36:43.125759+0000","last_peered":"2024-09-13T14:36:43.125759+0000","last_clean":"2024-09-13T14:36:43.125759+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:36:43.125759+0000","last_undegraded":"2024-09-13T14:36:43.125759+0000","last_fullsized":"2024-09-13T14:36:43.125759+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":396,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125618+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:36:43.125618+0000","last_peered":"2024-09-13T14:36:43.125618+0000","last_clean":"2024-09-13T14:36:43.125618+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:36:43.125618+0000","last_undegraded":"2024-09-13T14:36:43.125618+0000","last_fullsized":"2024-09-13T14:36:43.125618+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271858+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:36:40.271858+0000","last_peered":"2024-09-13T14:36:40.271858+0000","last_clean":"2024-09-13T14:36:40.271858+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:36:40.271858+0000","last_undegraded":"2024-09-13T14:36:40.271858+0000","last_fullsized":"2024-09-13T14:36:40.271858+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":412,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.394369+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:36:58.394369+0000","last_peered":"2024-09-13T14:36:58.394369+0000","last_clean":"2024-09-13T14:36:58.394369+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:36:58.394369+0000","last_undegraded":"2024-09-13T14:36:58.394369+0000","last_fullsized":"2024-09-13T14:36:58.394369+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":238,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.112813+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:36:43.112813+0000","last_peered":"2024-09-13T14:36:43.112813+0000","last_clean":"2024-09-13T14:36:43.112813+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:36:43.112813+0000","last_undegraded":"2024-09-13T14:36:43.112813+0000","last_fullsized":"2024-09-13T14:36:43.112813+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"266'8470","reported_seq":8519,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:59.579185+0000","last_change":"2024-09-13T14:34:57.524042+0000","last_active":"2024-09-13T14:36:59.579185+0000","last_peered":"2024-09-13T14:36:59.579185+0000","last_clean":"2024-09-13T14:36:59.579185+0000","last_became_active":"2024-09-13T14:34:57.523748+0000","last_became_peered":"2024-09-13T14:34:57.523748+0000","last_unstale":"2024-09-13T14:36:59.579185+0000","last_undegraded":"2024-09-13T14:36:59.579185+0000","last_fullsized":"2024-09-13T14:36:59.579185+0000","mapping_epoch":247,"log_start":"252'700","ondisk_log_start":"252'700","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:42.905684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421789696,"num_objects":6436,"num_object_clones":0,"num_object_copies":19308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8470,"num_write_kb":476992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":230,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272269+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:36:40.272269+0000","last_peered":"2024-09-13T14:36:40.272269+0000","last_clean":"2024-09-13T14:36:40.272269+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:36:40.272269+0000","last_undegraded":"2024-09-13T14:36:40.272269+0000","last_fullsized":"2024-09-13T14:36:40.272269+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271526+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:36:40.271526+0000","last_peered":"2024-09-13T14:36:40.271526+0000","last_clean":"2024-09-13T14:36:40.271526+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:36:40.271526+0000","last_undegraded":"2024-09-13T14:36:40.271526+0000","last_fullsized":"2024-09-13T14:36:40.271526+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"266'8531","reported_seq":8580,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:59.894831+0000","last_change":"2024-09-13T14:34:57.522300+0000","last_active":"2024-09-13T14:36:59.894831+0000","last_peered":"2024-09-13T14:36:59.894831+0000","last_clean":"2024-09-13T14:36:59.894831+0000","last_became_active":"2024-09-13T14:34:57.522005+0000","last_became_peered":"2024-09-13T14:34:57.522005+0000","last_unstale":"2024-09-13T14:36:59.894831+0000","last_undegraded":"2024-09-13T14:36:59.894831+0000","last_fullsized":"2024-09-13T14:36:59.894831+0000","mapping_epoch":247,"log_start":"252'1800","ondisk_log_start":"252'1800","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:56.018499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427884544,"num_objects":6529,"num_object_clones":0,"num_object_copies":19587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8531,"num_write_kb":481920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272249+0000","last_change":"2024-09-13T14:34:46.649053+0000","last_active":"2024-09-13T14:36:40.272249+0000","last_peered":"2024-09-13T14:36:40.272249+0000","last_clean":"2024-09-13T14:36:40.272249+0000","last_became_active":"2024-09-13T14:34:46.648725+0000","last_became_peered":"2024-09-13T14:34:46.648725+0000","last_unstale":"2024-09-13T14:36:40.272249+0000","last_undegraded":"2024-09-13T14:36:40.272249+0000","last_fullsized":"2024-09-13T14:36:40.272249+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:03.935325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":263,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.287615+0000","last_change":"2024-09-13T14:34:46.594184+0000","last_active":"2024-09-13T14:36:40.287615+0000","last_peered":"2024-09-13T14:36:40.287615+0000","last_clean":"2024-09-13T14:36:40.287615+0000","last_became_active":"2024-09-13T14:34:46.593911+0000","last_became_peered":"2024-09-13T14:34:46.593911+0000","last_unstale":"2024-09-13T14:36:40.287615+0000","last_undegraded":"2024-09-13T14:36:40.287615+0000","last_fullsized":"2024-09-13T14:36:40.287615+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:41.568032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'8344","reported_seq":8391,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.068609+0000","last_change":"2024-09-13T14:34:57.526403+0000","last_active":"2024-09-13T14:36:58.068609+0000","last_peered":"2024-09-13T14:36:58.068609+0000","last_clean":"2024-09-13T14:36:58.068609+0000","last_became_active":"2024-09-13T14:34:57.525671+0000","last_became_peered":"2024-09-13T14:34:57.525671+0000","last_unstale":"2024-09-13T14:36:58.068609+0000","last_undegraded":"2024-09-13T14:36:58.068609+0000","last_fullsized":"2024-09-13T14:36:58.068609+0000","mapping_epoch":247,"log_start":"251'200","ondisk_log_start":"251'200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:22.342551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428998656,"num_objects":6546,"num_object_clones":0,"num_object_copies":19638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8344,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":213,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272249+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:36:40.272249+0000","last_peered":"2024-09-13T14:36:40.272249+0000","last_clean":"2024-09-13T14:36:40.272249+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:36:40.272249+0000","last_undegraded":"2024-09-13T14:36:40.272249+0000","last_fullsized":"2024-09-13T14:36:40.272249+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.113066+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:36:43.113066+0000","last_peered":"2024-09-13T14:36:43.113066+0000","last_clean":"2024-09-13T14:36:43.113066+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:36:43.113066+0000","last_undegraded":"2024-09-13T14:36:43.113066+0000","last_fullsized":"2024-09-13T14:36:43.113066+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"266'8672","reported_seq":8721,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:37:01.509363+0000","last_change":"2024-09-13T14:34:57.521543+0000","last_active":"2024-09-13T14:37:01.509363+0000","last_peered":"2024-09-13T14:37:01.509363+0000","last_clean":"2024-09-13T14:37:01.509363+0000","last_became_active":"2024-09-13T14:34:57.521269+0000","last_became_peered":"2024-09-13T14:34:57.521269+0000","last_unstale":"2024-09-13T14:37:01.509363+0000","last_undegraded":"2024-09-13T14:37:01.509363+0000","last_fullsized":"2024-09-13T14:37:01.509363+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8672,"log_dups_size":0,"ondisk_log_size":8672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:21.067198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424542208,"num_objects":6478,"num_object_clones":0,"num_object_copies":19434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8672,"num_write_kb":484800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":229,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272098+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:36:40.272098+0000","last_peered":"2024-09-13T14:36:40.272098+0000","last_clean":"2024-09-13T14:36:40.272098+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:36:40.272098+0000","last_undegraded":"2024-09-13T14:36:40.272098+0000","last_fullsized":"2024-09-13T14:36:40.272098+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.126222+0000","last_change":"2024-09-13T14:34:46.616314+0000","last_active":"2024-09-13T14:36:43.126222+0000","last_peered":"2024-09-13T14:36:43.126222+0000","last_clean":"2024-09-13T14:36:43.126222+0000","last_became_active":"2024-09-13T14:34:46.611776+0000","last_became_peered":"2024-09-13T14:34:46.611776+0000","last_unstale":"2024-09-13T14:36:43.126222+0000","last_undegraded":"2024-09-13T14:36:43.126222+0000","last_fullsized":"2024-09-13T14:36:43.126222+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:44.754447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"265'8343","reported_seq":8390,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.057431+0000","last_change":"2024-09-13T14:34:57.526751+0000","last_active":"2024-09-13T14:36:58.057431+0000","last_peered":"2024-09-13T14:36:58.057431+0000","last_clean":"2024-09-13T14:36:58.057431+0000","last_became_active":"2024-09-13T14:34:57.526442+0000","last_became_peered":"2024-09-13T14:34:57.526442+0000","last_unstale":"2024-09-13T14:36:58.057431+0000","last_undegraded":"2024-09-13T14:36:58.057431+0000","last_fullsized":"2024-09-13T14:36:58.057431+0000","mapping_epoch":247,"log_start":"251'200","ondisk_log_start":"251'200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8143,"log_dups_size":0,"ondisk_log_size":8143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:09.948770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422772736,"num_objects":6451,"num_object_clones":0,"num_object_copies":19353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8343,"num_write_kb":473408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":376,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271561+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:36:40.271561+0000","last_peered":"2024-09-13T14:36:40.271561+0000","last_clean":"2024-09-13T14:36:40.271561+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:36:40.271561+0000","last_undegraded":"2024-09-13T14:36:40.271561+0000","last_fullsized":"2024-09-13T14:36:40.271561+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":399,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.383695+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:36:58.383695+0000","last_peered":"2024-09-13T14:36:58.383695+0000","last_clean":"2024-09-13T14:36:58.383695+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:36:58.383695+0000","last_undegraded":"2024-09-13T14:36:58.383695+0000","last_fullsized":"2024-09-13T14:36:58.383695+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":229,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285602+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:36:40.285602+0000","last_peered":"2024-09-13T14:36:40.285602+0000","last_clean":"2024-09-13T14:36:40.285602+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:36:40.285602+0000","last_undegraded":"2024-09-13T14:36:40.285602+0000","last_fullsized":"2024-09-13T14:36:40.285602+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"266'8555","reported_seq":8604,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:37:01.626644+0000","last_change":"2024-09-13T14:34:57.525034+0000","last_active":"2024-09-13T14:37:01.626644+0000","last_peered":"2024-09-13T14:37:01.626644+0000","last_clean":"2024-09-13T14:37:01.626644+0000","last_became_active":"2024-09-13T14:34:57.524740+0000","last_became_peered":"2024-09-13T14:34:57.524740+0000","last_unstale":"2024-09-13T14:37:01.626644+0000","last_undegraded":"2024-09-13T14:37:01.626644+0000","last_fullsized":"2024-09-13T14:37:01.626644+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8555,"log_dups_size":0,"ondisk_log_size":8555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:44.966638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417267712,"num_objects":6367,"num_object_clones":0,"num_object_copies":19101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8555,"num_write_kb":477504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":123,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271536+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:36:40.271536+0000","last_peered":"2024-09-13T14:36:40.271536+0000","last_clean":"2024-09-13T14:36:40.271536+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:36:40.271536+0000","last_undegraded":"2024-09-13T14:36:40.271536+0000","last_fullsized":"2024-09-13T14:36:40.271536+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.112921+0000","last_change":"2024-09-13T14:34:46.575576+0000","last_active":"2024-09-13T14:36:43.112921+0000","last_peered":"2024-09-13T14:36:43.112921+0000","last_clean":"2024-09-13T14:36:43.112921+0000","last_became_active":"2024-09-13T14:34:46.575291+0000","last_became_peered":"2024-09-13T14:34:46.575291+0000","last_unstale":"2024-09-13T14:36:43.112921+0000","last_undegraded":"2024-09-13T14:36:43.112921+0000","last_fullsized":"2024-09-13T14:36:43.112921+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:30.242110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285526+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:36:40.285526+0000","last_peered":"2024-09-13T14:36:40.285526+0000","last_clean":"2024-09-13T14:36:40.285526+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:36:40.285526+0000","last_undegraded":"2024-09-13T14:36:40.285526+0000","last_fullsized":"2024-09-13T14:36:40.285526+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"266'8615","reported_seq":8643,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:37:02.949025+0000","last_change":"2024-09-13T14:34:57.522071+0000","last_active":"2024-09-13T14:37:02.949025+0000","last_peered":"2024-09-13T14:37:02.949025+0000","last_clean":"2024-09-13T14:37:02.949025+0000","last_became_active":"2024-09-13T14:34:57.521798+0000","last_became_peered":"2024-09-13T14:34:57.521798+0000","last_unstale":"2024-09-13T14:37:02.949025+0000","last_undegraded":"2024-09-13T14:37:02.949025+0000","last_fullsized":"2024-09-13T14:37:02.949025+0000","mapping_epoch":247,"log_start":"251'500","ondisk_log_start":"251'500","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:47.723539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418512920,"num_objects":6387,"num_object_clones":0,"num_object_copies":19161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6387,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8615,"num_write_kb":480001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.287687+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:36:40.287687+0000","last_peered":"2024-09-13T14:36:40.287687+0000","last_clean":"2024-09-13T14:36:40.287687+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:36:40.287687+0000","last_undegraded":"2024-09-13T14:36:40.287687+0000","last_fullsized":"2024-09-13T14:36:40.287687+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285067+0000","last_change":"2024-09-13T14:34:46.648302+0000","last_active":"2024-09-13T14:36:40.285067+0000","last_peered":"2024-09-13T14:36:40.285067+0000","last_clean":"2024-09-13T14:36:40.285067+0000","last_became_active":"2024-09-13T14:34:46.647761+0000","last_became_peered":"2024-09-13T14:34:46.647761+0000","last_unstale":"2024-09-13T14:36:40.285067+0000","last_undegraded":"2024-09-13T14:36:40.285067+0000","last_fullsized":"2024-09-13T14:36:40.285067+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:22.055371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"266'8383","reported_seq":8410,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:37:02.934372+0000","last_change":"2024-09-13T14:34:57.521324+0000","last_active":"2024-09-13T14:37:02.934372+0000","last_peered":"2024-09-13T14:37:02.934372+0000","last_clean":"2024-09-13T14:37:02.934372+0000","last_became_active":"2024-09-13T14:34:57.520992+0000","last_became_peered":"2024-09-13T14:34:57.520992+0000","last_unstale":"2024-09-13T14:37:02.934372+0000","last_undegraded":"2024-09-13T14:37:02.934372+0000","last_fullsized":"2024-09-13T14:37:02.934372+0000","mapping_epoch":247,"log_start":"251'200","ondisk_log_start":"251'200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8183,"log_dups_size":0,"ondisk_log_size":8183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.105872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396427264,"num_objects":6049,"num_object_clones":0,"num_object_copies":18147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8383,"num_write_kb":461824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":116,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271663+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:36:40.271663+0000","last_peered":"2024-09-13T14:36:40.271663+0000","last_clean":"2024-09-13T14:36:40.271663+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:36:40.271663+0000","last_undegraded":"2024-09-13T14:36:40.271663+0000","last_fullsized":"2024-09-13T14:36:40.271663+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":414,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375435+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:36:58.375435+0000","last_peered":"2024-09-13T14:36:58.375435+0000","last_clean":"2024-09-13T14:36:58.375435+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:36:58.375435+0000","last_undegraded":"2024-09-13T14:36:58.375435+0000","last_fullsized":"2024-09-13T14:36:58.375435+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":362,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.382549+0000","last_change":"2024-09-13T14:34:46.624474+0000","last_active":"2024-09-13T14:36:58.382549+0000","last_peered":"2024-09-13T14:36:58.382549+0000","last_clean":"2024-09-13T14:36:58.382549+0000","last_became_active":"2024-09-13T14:34:46.624138+0000","last_became_peered":"2024-09-13T14:34:46.624138+0000","last_unstale":"2024-09-13T14:36:58.382549+0000","last_undegraded":"2024-09-13T14:36:58.382549+0000","last_fullsized":"2024-09-13T14:36:58.382549+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:51.252508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'8260","reported_seq":8307,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.057873+0000","last_change":"2024-09-13T14:34:57.525058+0000","last_active":"2024-09-13T14:36:58.057873+0000","last_peered":"2024-09-13T14:36:58.057873+0000","last_clean":"2024-09-13T14:36:58.057873+0000","last_became_active":"2024-09-13T14:34:57.524514+0000","last_became_peered":"2024-09-13T14:34:57.524514+0000","last_unstale":"2024-09-13T14:36:58.057873+0000","last_undegraded":"2024-09-13T14:36:58.057873+0000","last_fullsized":"2024-09-13T14:36:58.057873+0000","mapping_epoch":247,"log_start":"251'100","ondisk_log_start":"251'100","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:17.836817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427950080,"num_objects":6530,"num_object_clones":0,"num_object_copies":19590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8260,"num_write_kb":473280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":367,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271985+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:36:40.271985+0000","last_peered":"2024-09-13T14:36:40.271985+0000","last_clean":"2024-09-13T14:36:40.271985+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:36:40.271985+0000","last_undegraded":"2024-09-13T14:36:40.271985+0000","last_fullsized":"2024-09-13T14:36:40.271985+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":830,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272232+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:36:40.272232+0000","last_peered":"2024-09-13T14:36:40.272232+0000","last_clean":"2024-09-13T14:36:40.272232+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:36:40.272232+0000","last_undegraded":"2024-09-13T14:36:40.272232+0000","last_fullsized":"2024-09-13T14:36:40.272232+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":356,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.382259+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:36:58.382259+0000","last_peered":"2024-09-13T14:36:58.382259+0000","last_clean":"2024-09-13T14:36:58.382259+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:36:58.382259+0000","last_undegraded":"2024-09-13T14:36:58.382259+0000","last_fullsized":"2024-09-13T14:36:58.382259+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":212,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.118181+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:36:43.118181+0000","last_peered":"2024-09-13T14:36:43.118181+0000","last_clean":"2024-09-13T14:36:43.118181+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:36:43.118181+0000","last_undegraded":"2024-09-13T14:36:43.118181+0000","last_fullsized":"2024-09-13T14:36:43.118181+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"265'8372","reported_seq":8398,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.086298+0000","last_change":"2024-09-13T14:34:57.525197+0000","last_active":"2024-09-13T14:36:58.086298+0000","last_peered":"2024-09-13T14:36:58.086298+0000","last_clean":"2024-09-13T14:36:58.086298+0000","last_became_active":"2024-09-13T14:34:57.524916+0000","last_became_peered":"2024-09-13T14:34:57.524916+0000","last_unstale":"2024-09-13T14:36:58.086298+0000","last_undegraded":"2024-09-13T14:36:58.086298+0000","last_fullsized":"2024-09-13T14:36:58.086298+0000","mapping_epoch":247,"log_start":"252'600","ondisk_log_start":"252'600","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7772,"log_dups_size":0,"ondisk_log_size":7772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:01.282676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431751168,"num_objects":6588,"num_object_clones":0,"num_object_copies":19764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8372,"num_write_kb":478720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":201,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.385159+0000","last_change":"2024-09-13T14:34:47.477679+0000","last_active":"2024-09-13T14:36:58.385159+0000","last_peered":"2024-09-13T14:36:58.385159+0000","last_clean":"2024-09-13T14:36:58.385159+0000","last_became_active":"2024-09-13T14:34:46.634361+0000","last_became_peered":"2024-09-13T14:34:46.634361+0000","last_unstale":"2024-09-13T14:36:58.385159+0000","last_undegraded":"2024-09-13T14:36:58.385159+0000","last_fullsized":"2024-09-13T14:36:58.385159+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:44.197390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":655,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271935+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:36:40.271935+0000","last_peered":"2024-09-13T14:36:40.271935+0000","last_clean":"2024-09-13T14:36:40.271935+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:36:40.271935+0000","last_undegraded":"2024-09-13T14:36:40.271935+0000","last_fullsized":"2024-09-13T14:36:40.271935+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":396,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125327+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:36:43.125327+0000","last_peered":"2024-09-13T14:36:43.125327+0000","last_clean":"2024-09-13T14:36:43.125327+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:36:43.125327+0000","last_undegraded":"2024-09-13T14:36:43.125327+0000","last_fullsized":"2024-09-13T14:36:43.125327+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":419,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125798+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:36:43.125798+0000","last_peered":"2024-09-13T14:36:43.125798+0000","last_clean":"2024-09-13T14:36:43.125798+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:36:43.125798+0000","last_undegraded":"2024-09-13T14:36:43.125798+0000","last_fullsized":"2024-09-13T14:36:43.125798+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"265'8371","reported_seq":8418,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.069952+0000","last_change":"2024-09-13T14:34:57.525939+0000","last_active":"2024-09-13T14:36:58.069952+0000","last_peered":"2024-09-13T14:36:58.069952+0000","last_clean":"2024-09-13T14:36:58.069952+0000","last_became_active":"2024-09-13T14:34:57.525621+0000","last_became_peered":"2024-09-13T14:34:57.525621+0000","last_unstale":"2024-09-13T14:36:58.069952+0000","last_undegraded":"2024-09-13T14:36:58.069952+0000","last_fullsized":"2024-09-13T14:36:58.069952+0000","mapping_epoch":247,"log_start":"251'200","ondisk_log_start":"251'200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:26.471406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428539904,"num_objects":6539,"num_object_clones":0,"num_object_copies":19617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8371,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":376,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271637+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:36:40.271637+0000","last_peered":"2024-09-13T14:36:40.271637+0000","last_clean":"2024-09-13T14:36:40.271637+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:36:40.271637+0000","last_undegraded":"2024-09-13T14:36:40.271637+0000","last_fullsized":"2024-09-13T14:36:40.271637+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":827,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.380026+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:36:58.380026+0000","last_peered":"2024-09-13T14:36:58.380026+0000","last_clean":"2024-09-13T14:36:58.380026+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:36:58.380026+0000","last_undegraded":"2024-09-13T14:36:58.380026+0000","last_fullsized":"2024-09-13T14:36:58.380026+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":353,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.388506+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:36:58.388506+0000","last_peered":"2024-09-13T14:36:58.388506+0000","last_clean":"2024-09-13T14:36:58.388506+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:36:58.388506+0000","last_undegraded":"2024-09-13T14:36:58.388506+0000","last_fullsized":"2024-09-13T14:36:58.388506+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":404,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.380278+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:36:58.380278+0000","last_peered":"2024-09-13T14:36:58.380278+0000","last_clean":"2024-09-13T14:36:58.380278+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:36:58.380278+0000","last_undegraded":"2024-09-13T14:36:58.380278+0000","last_fullsized":"2024-09-13T14:36:58.380278+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"266'8703","reported_seq":8730,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:37:02.932217+0000","last_change":"2024-09-13T14:34:57.522448+0000","last_active":"2024-09-13T14:37:02.932217+0000","last_peered":"2024-09-13T14:37:02.932217+0000","last_clean":"2024-09-13T14:37:02.932217+0000","last_became_active":"2024-09-13T14:34:57.522167+0000","last_became_peered":"2024-09-13T14:34:57.522167+0000","last_unstale":"2024-09-13T14:37:02.932217+0000","last_undegraded":"2024-09-13T14:37:02.932217+0000","last_fullsized":"2024-09-13T14:37:02.932217+0000","mapping_epoch":247,"log_start":"251'500","ondisk_log_start":"251'500","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8203,"log_dups_size":0,"ondisk_log_size":8203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:22:47.802147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416088064,"num_objects":6349,"num_object_clones":0,"num_object_copies":19047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8703,"num_write_kb":481664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":116,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271534+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:36:40.271534+0000","last_peered":"2024-09-13T14:36:40.271534+0000","last_clean":"2024-09-13T14:36:40.271534+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:36:40.271534+0000","last_undegraded":"2024-09-13T14:36:40.271534+0000","last_fullsized":"2024-09-13T14:36:40.271534+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":896,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.271829+0000","last_change":"2024-09-13T14:34:46.566197+0000","last_active":"2024-09-13T14:36:40.271829+0000","last_peered":"2024-09-13T14:36:40.271829+0000","last_clean":"2024-09-13T14:36:40.271829+0000","last_became_active":"2024-09-13T14:34:46.565768+0000","last_became_peered":"2024-09-13T14:34:46.565768+0000","last_unstale":"2024-09-13T14:36:40.271829+0000","last_undegraded":"2024-09-13T14:36:40.271829+0000","last_fullsized":"2024-09-13T14:36:40.271829+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:12:46.443713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":230,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.286275+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:36:40.286275+0000","last_peered":"2024-09-13T14:36:40.286275+0000","last_clean":"2024-09-13T14:36:40.286275+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:36:40.286275+0000","last_undegraded":"2024-09-13T14:36:40.286275+0000","last_fullsized":"2024-09-13T14:36:40.286275+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":374,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.381857+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:36:58.381857+0000","last_peered":"2024-09-13T14:36:58.381857+0000","last_clean":"2024-09-13T14:36:58.381857+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:36:58.381857+0000","last_undegraded":"2024-09-13T14:36:58.381857+0000","last_fullsized":"2024-09-13T14:36:58.381857+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"265'8325","reported_seq":8351,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:57.966399+0000","last_change":"2024-09-13T14:34:57.521524+0000","last_active":"2024-09-13T14:36:57.966399+0000","last_peered":"2024-09-13T14:36:57.966399+0000","last_clean":"2024-09-13T14:36:57.966399+0000","last_became_active":"2024-09-13T14:34:57.520850+0000","last_became_peered":"2024-09-13T14:34:57.520850+0000","last_unstale":"2024-09-13T14:36:57.966399+0000","last_undegraded":"2024-09-13T14:36:57.966399+0000","last_fullsized":"2024-09-13T14:36:57.966399+0000","mapping_epoch":247,"log_start":"252'1000","ondisk_log_start":"252'1000","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:42.515897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429326336,"num_objects":6551,"num_object_clones":0,"num_object_copies":19653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8325,"num_write_kb":476032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":265,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.113526+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:36:43.113526+0000","last_peered":"2024-09-13T14:36:43.113526+0000","last_clean":"2024-09-13T14:36:43.113526+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:36:43.113526+0000","last_undegraded":"2024-09-13T14:36:43.113526+0000","last_fullsized":"2024-09-13T14:36:43.113526+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272410+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:36:40.272410+0000","last_peered":"2024-09-13T14:36:40.272410+0000","last_clean":"2024-09-13T14:36:40.272410+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:36:40.272410+0000","last_undegraded":"2024-09-13T14:36:40.272410+0000","last_fullsized":"2024-09-13T14:36:40.272410+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":953,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.382095+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:36:58.382095+0000","last_peered":"2024-09-13T14:36:58.382095+0000","last_clean":"2024-09-13T14:36:58.382095+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:36:58.382095+0000","last_undegraded":"2024-09-13T14:36:58.382095+0000","last_fullsized":"2024-09-13T14:36:58.382095+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":332,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.382038+0000","last_change":"2024-09-13T14:34:46.616754+0000","last_active":"2024-09-13T14:36:58.382038+0000","last_peered":"2024-09-13T14:36:58.382038+0000","last_clean":"2024-09-13T14:36:58.382038+0000","last_became_active":"2024-09-13T14:34:46.616476+0000","last_became_peered":"2024-09-13T14:34:46.616476+0000","last_unstale":"2024-09-13T14:36:58.382038+0000","last_undegraded":"2024-09-13T14:36:58.382038+0000","last_fullsized":"2024-09-13T14:36:58.382038+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:14.670350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"265'8402","reported_seq":8428,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.085047+0000","last_change":"2024-09-13T14:34:57.527395+0000","last_active":"2024-09-13T14:36:58.085047+0000","last_peered":"2024-09-13T14:36:58.085047+0000","last_clean":"2024-09-13T14:36:58.085047+0000","last_became_active":"2024-09-13T14:34:57.527133+0000","last_became_peered":"2024-09-13T14:34:57.527133+0000","last_unstale":"2024-09-13T14:36:58.085047+0000","last_undegraded":"2024-09-13T14:36:58.085047+0000","last_fullsized":"2024-09-13T14:36:58.085047+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8402,"log_dups_size":0,"ondisk_log_size":8402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:16.255103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426508288,"num_objects":6508,"num_object_clones":0,"num_object_copies":19524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8402,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":265,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.113158+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:36:43.113158+0000","last_peered":"2024-09-13T14:36:43.113158+0000","last_clean":"2024-09-13T14:36:43.113158+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:36:43.113158+0000","last_undegraded":"2024-09-13T14:36:43.113158+0000","last_fullsized":"2024-09-13T14:36:43.113158+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":353,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.381978+0000","last_change":"2024-09-13T14:34:46.583932+0000","last_active":"2024-09-13T14:36:58.381978+0000","last_peered":"2024-09-13T14:36:58.381978+0000","last_clean":"2024-09-13T14:36:58.381978+0000","last_became_active":"2024-09-13T14:34:46.580751+0000","last_became_peered":"2024-09-13T14:34:46.580751+0000","last_unstale":"2024-09-13T14:36:58.381978+0000","last_undegraded":"2024-09-13T14:36:58.381978+0000","last_fullsized":"2024-09-13T14:36:58.381978+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:38.905892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":575,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272356+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:36:40.272356+0000","last_peered":"2024-09-13T14:36:40.272356+0000","last_clean":"2024-09-13T14:36:40.272356+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:36:40.272356+0000","last_undegraded":"2024-09-13T14:36:40.272356+0000","last_fullsized":"2024-09-13T14:36:40.272356+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":386,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.126149+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:36:43.126149+0000","last_peered":"2024-09-13T14:36:43.126149+0000","last_clean":"2024-09-13T14:36:43.126149+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:36:43.126149+0000","last_undegraded":"2024-09-13T14:36:43.126149+0000","last_fullsized":"2024-09-13T14:36:43.126149+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"266'8247","reported_seq":8296,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:59.928338+0000","last_change":"2024-09-13T14:34:57.521462+0000","last_active":"2024-09-13T14:36:59.928338+0000","last_peered":"2024-09-13T14:36:59.928338+0000","last_clean":"2024-09-13T14:36:59.928338+0000","last_became_active":"2024-09-13T14:34:57.521178+0000","last_became_peered":"2024-09-13T14:34:57.521178+0000","last_unstale":"2024-09-13T14:36:59.928338+0000","last_undegraded":"2024-09-13T14:36:59.928338+0000","last_fullsized":"2024-09-13T14:36:59.928338+0000","mapping_epoch":247,"log_start":"252'1500","ondisk_log_start":"252'1500","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:32.574385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421330944,"num_objects":6429,"num_object_clones":0,"num_object_copies":19287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8247,"num_write_kb":469632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":116,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285354+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:36:40.285354+0000","last_peered":"2024-09-13T14:36:40.285354+0000","last_clean":"2024-09-13T14:36:40.285354+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:36:40.285354+0000","last_undegraded":"2024-09-13T14:36:40.285354+0000","last_fullsized":"2024-09-13T14:36:40.285354+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":400,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.394770+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:36:58.394770+0000","last_peered":"2024-09-13T14:36:58.394770+0000","last_clean":"2024-09-13T14:36:58.394770+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:36:58.394770+0000","last_undegraded":"2024-09-13T14:36:58.394770+0000","last_fullsized":"2024-09-13T14:36:58.394770+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":687,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.382251+0000","last_change":"2024-09-13T14:34:46.652006+0000","last_active":"2024-09-13T14:36:58.382251+0000","last_peered":"2024-09-13T14:36:58.382251+0000","last_clean":"2024-09-13T14:36:58.382251+0000","last_became_active":"2024-09-13T14:34:46.651532+0000","last_became_peered":"2024-09-13T14:34:46.651532+0000","last_unstale":"2024-09-13T14:36:58.382251+0000","last_undegraded":"2024-09-13T14:36:58.382251+0000","last_fullsized":"2024-09-13T14:36:58.382251+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:35.907330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":397,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375222+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:36:58.375222+0000","last_peered":"2024-09-13T14:36:58.375222+0000","last_clean":"2024-09-13T14:36:58.375222+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:36:58.375222+0000","last_undegraded":"2024-09-13T14:36:58.375222+0000","last_fullsized":"2024-09-13T14:36:58.375222+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"266'8429","reported_seq":8478,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:59.904189+0000","last_change":"2024-09-13T14:34:57.522594+0000","last_active":"2024-09-13T14:36:59.904189+0000","last_peered":"2024-09-13T14:36:59.904189+0000","last_clean":"2024-09-13T14:36:59.904189+0000","last_became_active":"2024-09-13T14:34:57.520039+0000","last_became_peered":"2024-09-13T14:34:57.520039+0000","last_unstale":"2024-09-13T14:36:59.904189+0000","last_undegraded":"2024-09-13T14:36:59.904189+0000","last_fullsized":"2024-09-13T14:36:59.904189+0000","mapping_epoch":247,"log_start":"252'1700","ondisk_log_start":"252'1700","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:37.564876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426311680,"num_objects":6505,"num_object_clones":0,"num_object_copies":19515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8429,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":238,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.383893+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:36:58.383893+0000","last_peered":"2024-09-13T14:36:58.383893+0000","last_clean":"2024-09-13T14:36:58.383893+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:36:58.383893+0000","last_undegraded":"2024-09-13T14:36:58.383893+0000","last_fullsized":"2024-09-13T14:36:58.383893+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":411,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125572+0000","last_change":"2024-09-13T14:34:46.577789+0000","last_active":"2024-09-13T14:36:43.125572+0000","last_peered":"2024-09-13T14:36:43.125572+0000","last_clean":"2024-09-13T14:36:43.125572+0000","last_became_active":"2024-09-13T14:34:46.574952+0000","last_became_peered":"2024-09-13T14:34:46.574952+0000","last_unstale":"2024-09-13T14:36:43.125572+0000","last_undegraded":"2024-09-13T14:36:43.125572+0000","last_fullsized":"2024-09-13T14:36:43.125572+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:20:04.720034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":684,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.113663+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:36:43.113663+0000","last_peered":"2024-09-13T14:36:43.113663+0000","last_clean":"2024-09-13T14:36:43.113663+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:36:43.113663+0000","last_undegraded":"2024-09-13T14:36:43.113663+0000","last_fullsized":"2024-09-13T14:36:43.113663+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.287494+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:36:40.287494+0000","last_peered":"2024-09-13T14:36:40.287494+0000","last_clean":"2024-09-13T14:36:40.287494+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:36:40.287494+0000","last_undegraded":"2024-09-13T14:36:40.287494+0000","last_fullsized":"2024-09-13T14:36:40.287494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.286147+0000","last_change":"2024-09-13T14:34:46.645586+0000","last_active":"2024-09-13T14:36:40.286147+0000","last_peered":"2024-09-13T14:36:40.286147+0000","last_clean":"2024-09-13T14:36:40.286147+0000","last_became_active":"2024-09-13T14:34:46.645294+0000","last_became_peered":"2024-09-13T14:34:46.645294+0000","last_unstale":"2024-09-13T14:36:40.286147+0000","last_undegraded":"2024-09-13T14:36:40.286147+0000","last_fullsized":"2024-09-13T14:36:40.286147+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:55.643942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285684+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:36:40.285684+0000","last_peered":"2024-09-13T14:36:40.285684+0000","last_clean":"2024-09-13T14:36:40.285684+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:36:40.285684+0000","last_undegraded":"2024-09-13T14:36:40.285684+0000","last_fullsized":"2024-09-13T14:36:40.285684+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":351,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.381894+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:36:58.381894+0000","last_peered":"2024-09-13T14:36:58.381894+0000","last_clean":"2024-09-13T14:36:58.381894+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:36:58.381894+0000","last_undegraded":"2024-09-13T14:36:58.381894+0000","last_fullsized":"2024-09-13T14:36:58.381894+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272116+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:36:40.272116+0000","last_peered":"2024-09-13T14:36:40.272116+0000","last_clean":"2024-09-13T14:36:40.272116+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:36:40.272116+0000","last_undegraded":"2024-09-13T14:36:40.272116+0000","last_fullsized":"2024-09-13T14:36:40.272116+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272515+0000","last_change":"2024-09-13T14:34:46.565021+0000","last_active":"2024-09-13T14:36:40.272515+0000","last_peered":"2024-09-13T14:36:40.272515+0000","last_clean":"2024-09-13T14:36:40.272515+0000","last_became_active":"2024-09-13T14:34:46.563784+0000","last_became_peered":"2024-09-13T14:34:46.563784+0000","last_unstale":"2024-09-13T14:36:40.272515+0000","last_undegraded":"2024-09-13T14:36:40.272515+0000","last_fullsized":"2024-09-13T14:36:40.272515+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:00.380107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":356,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.374652+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:36:58.374652+0000","last_peered":"2024-09-13T14:36:58.374652+0000","last_clean":"2024-09-13T14:36:58.374652+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:36:58.374652+0000","last_undegraded":"2024-09-13T14:36:58.374652+0000","last_fullsized":"2024-09-13T14:36:58.374652+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":395,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.138104+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:36:43.138104+0000","last_peered":"2024-09-13T14:36:43.138104+0000","last_clean":"2024-09-13T14:36:43.138104+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:36:43.138104+0000","last_undegraded":"2024-09-13T14:36:43.138104+0000","last_fullsized":"2024-09-13T14:36:43.138104+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":412,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375189+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:36:58.375189+0000","last_peered":"2024-09-13T14:36:58.375189+0000","last_clean":"2024-09-13T14:36:58.375189+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:36:58.375189+0000","last_undegraded":"2024-09-13T14:36:58.375189+0000","last_fullsized":"2024-09-13T14:36:58.375189+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":246,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.113593+0000","last_change":"2024-09-13T14:34:46.570197+0000","last_active":"2024-09-13T14:36:43.113593+0000","last_peered":"2024-09-13T14:36:43.113593+0000","last_clean":"2024-09-13T14:36:43.113593+0000","last_became_active":"2024-09-13T14:34:46.568152+0000","last_became_peered":"2024-09-13T14:34:46.568152+0000","last_unstale":"2024-09-13T14:36:43.113593+0000","last_undegraded":"2024-09-13T14:36:43.113593+0000","last_fullsized":"2024-09-13T14:36:43.113593+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:24:44.592857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":430,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375337+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:36:58.375337+0000","last_peered":"2024-09-13T14:36:58.375337+0000","last_clean":"2024-09-13T14:36:58.375337+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:36:58.375337+0000","last_undegraded":"2024-09-13T14:36:58.375337+0000","last_fullsized":"2024-09-13T14:36:58.375337+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":397,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.125694+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:36:43.125694+0000","last_peered":"2024-09-13T14:36:43.125694+0000","last_clean":"2024-09-13T14:36:43.125694+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:36:43.125694+0000","last_undegraded":"2024-09-13T14:36:43.125694+0000","last_fullsized":"2024-09-13T14:36:43.125694+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":362,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.385632+0000","last_change":"2024-09-13T14:34:46.624853+0000","last_active":"2024-09-13T14:36:58.385632+0000","last_peered":"2024-09-13T14:36:58.385632+0000","last_clean":"2024-09-13T14:36:58.385632+0000","last_became_active":"2024-09-13T14:34:46.624553+0000","last_became_peered":"2024-09-13T14:34:46.624553+0000","last_unstale":"2024-09-13T14:36:58.385632+0000","last_undegraded":"2024-09-13T14:36:58.385632+0000","last_fullsized":"2024-09-13T14:36:58.385632+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:22.679066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":412,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.385903+0000","last_change":"2024-09-13T14:34:46.567930+0000","last_active":"2024-09-13T14:36:58.385903+0000","last_peered":"2024-09-13T14:36:58.385903+0000","last_clean":"2024-09-13T14:36:58.385903+0000","last_became_active":"2024-09-13T14:34:46.567500+0000","last_became_peered":"2024-09-13T14:34:46.567500+0000","last_unstale":"2024-09-13T14:36:58.385903+0000","last_undegraded":"2024-09-13T14:36:58.385903+0000","last_fullsized":"2024-09-13T14:36:58.385903+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:47.997991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":265,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285227+0000","last_change":"2024-09-13T14:34:46.581764+0000","last_active":"2024-09-13T14:36:40.285227+0000","last_peered":"2024-09-13T14:36:40.285227+0000","last_clean":"2024-09-13T14:36:40.285227+0000","last_became_active":"2024-09-13T14:34:46.581377+0000","last_became_peered":"2024-09-13T14:34:46.581377+0000","last_unstale":"2024-09-13T14:36:40.285227+0000","last_undegraded":"2024-09-13T14:36:40.285227+0000","last_fullsized":"2024-09-13T14:36:40.285227+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:28.920200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":432,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.381626+0000","last_change":"2024-09-13T14:34:46.582553+0000","last_active":"2024-09-13T14:36:58.381626+0000","last_peered":"2024-09-13T14:36:58.381626+0000","last_clean":"2024-09-13T14:36:58.381626+0000","last_became_active":"2024-09-13T14:34:46.582146+0000","last_became_peered":"2024-09-13T14:34:46.582146+0000","last_unstale":"2024-09-13T14:36:58.381626+0000","last_undegraded":"2024-09-13T14:36:58.381626+0000","last_fullsized":"2024-09-13T14:36:58.381626+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:55:13.365210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":388,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.126052+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:36:43.126052+0000","last_peered":"2024-09-13T14:36:43.126052+0000","last_clean":"2024-09-13T14:36:43.126052+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:36:43.126052+0000","last_undegraded":"2024-09-13T14:36:43.126052+0000","last_fullsized":"2024-09-13T14:36:43.126052+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":263,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.286080+0000","last_change":"2024-09-13T14:34:46.584115+0000","last_active":"2024-09-13T14:36:40.286080+0000","last_peered":"2024-09-13T14:36:40.286080+0000","last_clean":"2024-09-13T14:36:40.286080+0000","last_became_active":"2024-09-13T14:34:46.583779+0000","last_became_peered":"2024-09-13T14:34:46.583779+0000","last_unstale":"2024-09-13T14:36:40.286080+0000","last_undegraded":"2024-09-13T14:36:40.286080+0000","last_fullsized":"2024-09-13T14:36:40.286080+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:19.009728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272024+0000","last_change":"2024-09-13T14:34:46.663283+0000","last_active":"2024-09-13T14:36:40.272024+0000","last_peered":"2024-09-13T14:36:40.272024+0000","last_clean":"2024-09-13T14:36:40.272024+0000","last_became_active":"2024-09-13T14:34:46.662866+0000","last_became_peered":"2024-09-13T14:34:46.662866+0000","last_unstale":"2024-09-13T14:36:40.272024+0000","last_undegraded":"2024-09-13T14:36:40.272024+0000","last_fullsized":"2024-09-13T14:36:40.272024+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:17:19.510340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":246,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.285152+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:36:40.285152+0000","last_peered":"2024-09-13T14:36:40.285152+0000","last_clean":"2024-09-13T14:36:40.285152+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:36:40.285152+0000","last_undegraded":"2024-09-13T14:36:40.285152+0000","last_fullsized":"2024-09-13T14:36:40.285152+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":214,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.272381+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:36:40.272381+0000","last_peered":"2024-09-13T14:36:40.272381+0000","last_clean":"2024-09-13T14:36:40.272381+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:36:40.272381+0000","last_undegraded":"2024-09-13T14:36:40.272381+0000","last_fullsized":"2024-09-13T14:36:40.272381+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":422,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375186+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:36:58.375186+0000","last_peered":"2024-09-13T14:36:58.375186+0000","last_clean":"2024-09-13T14:36:58.375186+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:36:58.375186+0000","last_undegraded":"2024-09-13T14:36:58.375186+0000","last_fullsized":"2024-09-13T14:36:58.375186+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":408,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.383403+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:36:58.383403+0000","last_peered":"2024-09-13T14:36:58.383403+0000","last_clean":"2024-09-13T14:36:58.383403+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:36:58.383403+0000","last_undegraded":"2024-09-13T14:36:58.383403+0000","last_fullsized":"2024-09-13T14:36:58.383403+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":239,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:36:40.284915+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:36:40.284915+0000","last_peered":"2024-09-13T14:36:40.284915+0000","last_clean":"2024-09-13T14:36:40.284915+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:36:40.284915+0000","last_undegraded":"2024-09-13T14:36:40.284915+0000","last_fullsized":"2024-09-13T14:36:40.284915+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":400,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:36:43.129850+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:36:43.129850+0000","last_peered":"2024-09-13T14:36:43.129850+0000","last_clean":"2024-09-13T14:36:43.129850+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:36:43.129850+0000","last_undegraded":"2024-09-13T14:36:43.129850+0000","last_fullsized":"2024-09-13T14:36:43.129850+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6766002200,"num_objects":103242,"num_object_clones":0,"num_object_copies":309726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103242,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135022,"num_write_kb":7624385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20339240960,"data_stored":20339228744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125822,"ondisk_log_size":125822,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397810,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2397076,"kb_used_data":2095564,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91348076,"statfs":{"total":95995035648,"available":93540429824,"internally_reserved":0,"allocated":2145857536,"data_stored":2144541730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594039,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323092,"kb_used_data":2017420,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91422060,"statfs":{"total":95995035648,"available":93616189440,"internally_reserved":0,"allocated":2065838080,"data_stored":2064529938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790268,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1996928,"kb_used_data":1687096,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91748224,"statfs":{"total":95995035648,"available":93950181376,"internally_reserved":0,"allocated":1727586304,"data_stored":1726286591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986496,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2412816,"kb_used_data":2111304,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91332336,"statfs":{"total":95995035648,"available":93524312064,"internally_reserved":0,"allocated":2161975296,"data_stored":2160663516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697054,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4308876,"kb_used_data":3730692,"kb_used_omap":4,"kb_used_meta":578043,"kb_avail":89436276,"statfs":{"total":95995035648,"available":91582746624,"internally_reserved":0,"allocated":3820228608,"data_stored":3818925014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":591916107},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860581,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2902416,"kb_used_data":2465800,"kb_used_omap":8,"kb_used_meta":436599,"kb_avail":90842736,"statfs":{"total":95995035648,"available":93022961664,"internally_reserved":0,"allocated":2524979200,"data_stored":2523672032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":447078220},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020252,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2610832,"kb_used_data":2499076,"kb_used_omap":9,"kb_used_meta":111734,"kb_avail":91134320,"statfs":{"total":95995035648,"available":93321543680,"internally_reserved":0,"allocated":2559053824,"data_stored":2557749639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":114416453},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187642,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3827072,"kb_used_data":3366520,"kb_used_omap":7,"kb_used_meta":460536,"kb_avail":89918080,"statfs":{"total":95995035648,"available":92076113920,"internally_reserved":0,"allocated":3447316480,"data_stored":3446016515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":471589195},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3432779776,"data_stored":3432775704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2544308224,"data_stored":2544304152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2518220800,"data_stored":2518220800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3787784192,"data_stored":3787784192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2151284736,"data_stored":2151284736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1718747136,"data_stored":1718747136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2053836800,"data_stored":2053832728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2132279296,"data_stored":2132279296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:37:05.047 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:37:05.048 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:37:05.048 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-13T14:37:05.048 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:37:05.048 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:37:05.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-13T14:37:05.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:05.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:05.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:05.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:05.322 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:37:05.323 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:37:05.377 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:37:05.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:05.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:05.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:05.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:05.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:05 smithi017 ceph-mon[103001]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:05.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:05 smithi017 ceph-mon[107556]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:05.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:05 smithi189 ceph-mon[90992]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:05.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:05.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:37:05.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:05.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:06.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:06.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:06.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:06.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:06.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:06.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:06.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:06.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:37:06.411 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:06.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:06.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:06.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:06.642 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:06 smithi189 ceph-mon[90992]: pgmap v754: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2024-09-13T14:37:06.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:06.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:06 smithi017 ceph-mon[103001]: pgmap v754: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2024-09-13T14:37:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:06 smithi017 ceph-mon[107556]: pgmap v754: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2024-09-13T14:37:06.780 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:06.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:06.912 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:06.943 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:06.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:07.045 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:07.050 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:07.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:07.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:07.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:07.266 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:37:07.267 INFO:teuthology.orchestra.run.smithi189.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:37:07.267 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:37:07.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:07.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:07.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:07.462 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:07.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:07.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:07.641 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:07] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:37:07.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:07.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:07.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:07.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:07.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:07.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:07.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:08.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:08.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:08.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:08.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:08.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:08.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:08.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:08.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:08.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:08.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:08.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:08.682 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:08 smithi017 ceph-mon[103001]: pgmap v755: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:08 smithi017 ceph-mon[107556]: pgmap v755: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:08.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:08.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:08.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:08.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:08.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:08.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:08.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:08 smithi189 ceph-mon[90992]: pgmap v755: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:09.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:09.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:09.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:09.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:09.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:09.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:09.322 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:09.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:09.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:09.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:09.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:09.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:09.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:09.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:09.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:09.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:09.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:10.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:10.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:10.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:10.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:10.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:10.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:10.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:10.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:10.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:10.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:10.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:10.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:10 smithi017 ceph-mon[103001]: pgmap v756: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-09-13T14:37:10.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:10 smithi017 ceph-mon[107556]: pgmap v756: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-09-13T14:37:10.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:10.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:10.817 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:10 smithi189 ceph-mon[90992]: pgmap v756: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-09-13T14:37:10.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:10.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:10.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:10.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:11.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:11.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:11.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:11.145 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:37:11.145 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:37:11.195 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:37:11.196 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:37:11.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:11.251 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:37:11.251 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:37:11.255 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:37:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=cleanup t=2024-09-13T14:37:10.816851243Z level=info msg="Completed cleanup jobs" duration=13.413084ms 2024-09-13T14:37:11.263 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:11.308 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:37:11.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:37:11.352 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:37:11.353 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:37:11.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:11.392 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:37:11.392 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:37:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:11.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:11.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:37:11.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:37:11.464 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:37:11.465 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:37:11.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:11.541 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:37:11.554 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:11.656 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:11.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:11.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1167352795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2384523857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1167352795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:11.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2384523857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1167352795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:11.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2384523857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:37:11.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:11.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:11.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:11.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:11.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:12.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:12.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:12.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:12.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:12.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:12.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:12.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:12.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:12.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:12.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:12.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:12.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[103001]: pgmap v757: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-09-13T14:37:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:37:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[103001]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:37:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[107556]: pgmap v757: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-09-13T14:37:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:37:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[107556]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:37:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:12.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:12.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:12.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:12.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:12.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:12.973 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:37:12.973 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:37:12.974 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:37:12.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:37:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:12 smithi189 ceph-mon[90992]: pgmap v757: 99 pgs: 99 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-09-13T14:37:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:12 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:12 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:37:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:12 smithi189 ceph-mon[90992]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:37:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:13.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:13.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:13.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:13.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:13.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:13.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:13.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:13.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:13.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:13.659 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:13.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:13.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:13.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:13.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:13.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:14.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:14.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:14.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:14.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:14.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:14.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:14.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:14.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:14.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:14.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:14.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:14.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:14.870 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:14.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:14.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:14.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:14.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:14 smithi017 ceph-mon[103001]: pgmap v759: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-09-13T14:37:14.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:14 smithi017 ceph-mon[107556]: pgmap v759: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-09-13T14:37:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:14 smithi189 ceph-mon[90992]: pgmap v759: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-09-13T14:37:15.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:15.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:15.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:15.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:15.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:15.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:15.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:15.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:37:15.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:15.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:15.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:15.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:15.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:15.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:15.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:15.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:15.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:16.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:16.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:16.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:16.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:37:16.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:16.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:16.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:16.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:16.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:16.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:16.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:16.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:16 smithi017 ceph-mon[107556]: pgmap v760: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:16.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:16 smithi017 ceph-mon[103001]: pgmap v760: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:16.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:16.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:16.899 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:16.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:16.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:16.968 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:37:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:27926 7.b 7:d02907a4:::benchmark_data_smithi017_220393_object27925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:35:21.387021+0000", 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 115.576954343, 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.061951052999999999, 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 27926 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387021+0000", 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387021+0000", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387023+0000", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9530000000000002e-06 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387069+0000", 2024-09-13T14:37:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6156999999999999e-05 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387090+0000", 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0369000000000001e-05 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387099+0000", 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387165+0000", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6859000000000005e-05 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387297+0000", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131089 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.387461+0000", 2024-09-13T14:37:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016409199999999999 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.388554+0000", 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010932170000000001 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.388785+0000", 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023061699999999999 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.448853+0000", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.060068605999999997 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.448927+0000", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3761000000000001e-05 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:35:21.448972+0000", 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5212e-05 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95741 7.9 e259/247 7:95db0216:::benchmark_data_smithi017_220393_object95740:head v 259'5969, mlcod=259'5966)", 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.960106+0000", 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.003869358999999, 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.083903963999999998, 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960106+0000", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960106+0000", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960107+0000", 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9699999999999996e-07 2024-09-13T14:37:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960132+0000", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5443e-05 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960146+0000", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.347e-05 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960152+0000", 2024-09-13T14:37:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8220000000000001e-06 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960233+0000", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0625999999999997e-05 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.960254+0000", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1049000000000001e-05 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043955+0000", 2024-09-13T14:37:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.083701094000000004 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044010+0000", 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4863e-05 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95751 7.9 e259/247 7:9ec6ee88:::benchmark_data_smithi017_220393_object95750:head v 259'5970, mlcod=259'5966)", 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.967011+0000", 2024-09-13T14:37:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.996964282, 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079955196000000006, 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967011+0000", 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967011+0000", 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967012+0000", 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7499999999999999e-07 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967042+0000", 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0076000000000001e-05 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967054+0000", 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1902999999999999e-05 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967062+0000", 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0509999999999997e-06 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967722+0000", 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00066035499999999995 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.967740+0000", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7696000000000001e-05 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.046855+0000", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079115446000000006 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.046966+0000", 2024-09-13T14:37:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110894 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95753 7.9 e259/247 7:94cfdce6:::benchmark_data_smithi017_220393_object95752:head v 259'5971, mlcod=259'5966)", 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.969386+0000", 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.994589490000003, 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077664403000000007, 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969386+0000", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969386+0000", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969387+0000", 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.099e-06 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969412+0000", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5130000000000002e-05 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969424+0000", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.145e-05 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.969431+0000", 2024-09-13T14:37:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.362e-06 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.970748+0000", 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013168640000000001 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.970760+0000", 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1926999999999999e-05 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.046973+0000", 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.076213227999999994 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.047050+0000", 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7342999999999995e-05 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95759 7.9 e259/247 7:92107978:::benchmark_data_smithi017_220393_object95758:head v 259'5972, mlcod=259'5967)", 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.971658+0000", 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.992317831000001, 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079618323000000005, 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971658+0000", 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971658+0000", 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971659+0000", 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031e-06 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971681+0000", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2218e-05 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971691+0000", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0635000000000001e-05 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971698+0000", 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3799999999999999e-06 2024-09-13T14:37:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044045+0000", 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.072347011000000003 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044064+0000", 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9196000000000001e-05 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.051067+0000", 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070030149999999996 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.051276+0000", 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020883699999999999 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95757 7.d e259/247 7:ba3ad13e:::benchmark_data_smithi017_220393_object95756:head v 259'6005, mlcod=259'6003)", 2024-09-13T14:37:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.971972+0000", 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.992003072999999, 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077773989000000002, 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971972+0000", 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971972+0000", 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.971973+0000", 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9899999999999999e-07 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.972035+0000", 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2069999999999994e-05 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.972046+0000", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0964e-05 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.972053+0000", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6880000000000002e-06 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043980+0000", 2024-09-13T14:37:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.071927141 2024-09-13T14:37:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044007+0000", 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6692000000000001e-05 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.049696+0000", 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056893730000000002 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.049746+0000", 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0262000000000001e-05 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95765 7.9 e259/247 7:9287f147:::benchmark_data_smithi017_220393_object95764:head v 259'5973, mlcod=259'5967)", 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.976726+0000", 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.987249315, 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074709302000000005, 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.976726+0000", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.976726+0000", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.976727+0000", 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3319999999999999e-06 2024-09-13T14:37:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.976773+0000", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5862999999999998e-05 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043954+0000", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.067180891000000006 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043968+0000", 2024-09-13T14:37:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3933e-05 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.047093+0000", 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003124894 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.047114+0000", 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1121e-05 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.051281+0000", 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041668410000000001 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.051435+0000", 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000154427 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:95771 7.2 e259/247 7:40dbe8cf:::benchmark_data_smithi017_220393_object95770:head v 259'6028, mlcod=259'6026)", 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:13.987783+0000", 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 62.976192273000002, 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.070407676000000002, 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.987783+0000", 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.987783+0000", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.987784+0000", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.297e-06 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:13.987814+0000", 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0006999999999999e-05 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043958+0000", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.056143242000000003 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.043969+0000", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0886e-05 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044035+0000", 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6484999999999998e-05 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.044056+0000", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1073999999999998e-05 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.058122+0000", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014065803 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:14.058191+0000", 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8881999999999995e-05 2024-09-13T14:37:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:112251 7.b 7:ddd89118:::benchmark_data_smithi017_220393_object112250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:32.395319+0000", 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.568656091000001, 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.052393149, 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 112251 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395319+0000", 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395319+0000", 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395321+0000", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7659999999999999e-06 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395346+0000", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4649000000000001e-05 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395363+0000", 2024-09-13T14:37:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.749e-05 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.395371+0000", 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4109999999999999e-06 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.405324+0000", 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0099534110000000005 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.405463+0000", 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000139357 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.405642+0000", 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017821499999999999 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.417197+0000", 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011555070000000001 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.417243+0000", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6097999999999998e-05 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.447543+0000", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.030300417999999999 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.447647+0000", 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104108 2024-09-13T14:37:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:32.447712+0000", 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5155999999999998e-05 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:128358 7.b 7:df501339:::benchmark_data_smithi017_220393_object9224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.269843+0000", 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.694131936000002, 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.095377584000000001, 2024-09-13T14:37:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 128358 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269843+0000", 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269843+0000", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269845+0000", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6360000000000001e-06 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269851+0000", 2024-09-13T14:37:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8440000000000003e-06 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269864+0000", 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3203999999999999e-05 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269872+0000", 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6680000000000001e-06 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.270121+0000", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000249328 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.270231+0000", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011024200000000001 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.270338+0000", 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010646599999999999 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280889+0000", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010551331000000001 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280931+0000", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1965999999999998e-05 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365070+0000", 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.084138881999999998 2024-09-13T14:37:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365173+0000", 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102932 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365221+0000", 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8084999999999998e-05 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:128359 7.b 7:d39b088f:::benchmark_data_smithi017_220393_object9225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.269900+0000", 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.694075390999998, 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.095518186000000005, 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 128359 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269900+0000", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269900+0000", 2024-09-13T14:37:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269900+0000", 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4999999999999998e-07 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269903+0000", 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4439999999999998e-06 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269910+0000", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9750000000000001e-06 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.269915+0000", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3469999999999998e-06 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.276058+0000", 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061430549999999997 2024-09-13T14:37:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.276163+0000", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010438999999999999 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.276269+0000", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010623 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280968+0000", 2024-09-13T14:37:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004698944 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363330+0000", 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.082362350000000001 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365231+0000", 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001900994 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365284+0000", 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2874000000000002e-05 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365418+0000", 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013413300000000001 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:128366 7.b 7:dc67bc74:::benchmark_data_smithi017_220393_object9232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:37:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.273984+0000", 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.689991737, 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.097475180999999994, 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 128366 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.273984+0000", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.273984+0000", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.273985+0000", 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.477e-06 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.273991+0000", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6529999999999999e-06 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.274004+0000", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2979000000000001e-05 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.274012+0000", 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6950000000000006e-06 2024-09-13T14:37:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280995+0000", 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069824070000000004 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.281094+0000", 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9227000000000004e-05 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.281194+0000", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9820000000000003e-05 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365048+0000", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.083854222000000006 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365442+0000", 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039390600000000002 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.371005+0000", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00556275 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.371263+0000", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000257922 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.371459+0000", 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019612300000000001 2024-09-13T14:37:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:128367 7.9 e265/247 7:92b57b05:::benchmark_data_smithi017_220393_object9233:head v 265'8004, mlcod=265'8001)", 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.278917+0000", 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.685058474000002, 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.084316247999999996, 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278917+0000", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278917+0000", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278918+0000", 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2399999999999997e-07 2024-09-13T14:37:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278923+0000", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.882e-06 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278935+0000", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2480000000000001e-05 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.278943+0000", 2024-09-13T14:37:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.503e-06 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280288+0000", 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013454199999999999 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280306+0000", 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7730000000000001e-05 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283923+0000", 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036168979999999999 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363233+0000", 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079310511 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:128370 7.4 e265/247 7:2cd5b3e7:::benchmark_data_smithi017_220393_object9236:head v 265'8043, mlcod=265'8041)", 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.280759+0000", 2024-09-13T14:37:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.683216099999999, 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.082492550999999997, 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280759+0000", 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280759+0000", 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280760+0000", 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6000000000000004e-07 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:17.042 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:16 smithi189 ceph-mon[90992]: pgmap v760: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280765+0000", 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4060000000000004e-06 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280775+0000", 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5480000000000007e-06 2024-09-13T14:37:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280781+0000", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7009999999999999e-06 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280914+0000", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013297299999999999 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.280934+0000", 2024-09-13T14:37:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9933999999999999e-05 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363212+0000", 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.082278022000000006 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363252+0000", 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9406999999999997e-05 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45566.0:128359 7.b e265/247 ondisk, result = 0)", 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.283827+0000", 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.680148012, 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079513326999999995, 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283827+0000", 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283827+0000", 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283828+0000", 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.04e-06 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283834+0000", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.541e-06 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283845+0000", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0968000000000001e-05 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283853+0000", 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2619999999999998e-06 2024-09-13T14:37:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363315+0000", 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079461754999999995 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363323+0000", 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5199999999999997e-06 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363341+0000", 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7241e-05 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:128371 7.b 7:d49e13dd:::benchmark_data_smithi017_220393_object9237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.283946+0000", 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.680029824999998, 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.089466727999999995, 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 128371 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283946+0000", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283946+0000", 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283947+0000", 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4619999999999999e-06 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.283952+0000", 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7620000000000003e-06 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363209+0000", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079257600999999997 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363219+0000", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0108e-05 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365511+0000", 2024-09-13T14:37:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022920029999999999 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365624+0000", 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112283 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365735+0000", 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110951 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.371613+0000", 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058782979999999997 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.372692+0000", 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001079216 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.373009+0000", 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031681800000000001 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.373267+0000", 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025840699999999999 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.373412+0000", 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144819 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:128375 7.9 e265/247 7:93812455:::benchmark_data_smithi017_220393_object9241:head v 265'8005, mlcod=265'8002)", 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.285534+0000", 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.678440888000001, 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.092232282999999998, 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.285534+0000", 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.285534+0000", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.285535+0000", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0800000000000004e-07 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.285540+0000", 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0429999999999998e-06 2024-09-13T14:37:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363186+0000", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077645643 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363203+0000", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7041000000000002e-05 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363286+0000", 2024-09-13T14:37:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2826999999999998e-05 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363311+0000", 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4729e-05 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.377679+0000", 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014368881999999999 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.377767+0000", 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7310000000000003e-05 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45566.0:128366 7.b e265/247 ondisk, result = 0)", 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:51.288203+0000", 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.675771982000001, 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077241331999999996, 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.288203+0000", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.288203+0000", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.288205+0000", 2024-09-13T14:37:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1999999999999999e-06 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.288213+0000", 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5820000000000002e-06 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363203+0000", 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074989569000000006 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.363211+0000", 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000001e-06 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.364999+0000", 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017880820000000001 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365040+0000", 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0429999999999997e-05 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:51.365445+0000", 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040480900000000001 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:130926 7.d e265/247 7:ba6daf1f:::benchmark_data_smithi017_220393_object11792:head v 265'8248, mlcod=265'8243)", 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:36:54.416096+0000", 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 22.547879229999999, 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051339924000000002, 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416096+0000", 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416096+0000", 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416097+0000", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3900000000000004e-07 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416103+0000", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0809999999999999e-06 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416113+0000", 2024-09-13T14:37:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7270000000000002e-06 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.416120+0000", 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7700000000000004e-06 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.444047+0000", 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027927746 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.444067+0000", 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0024999999999999e-05 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.467271+0000", 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023203970000000001 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:36:54.467436+0000", 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000164766 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:144714 7.0 e267/247 7:0ce963c0:::benchmark_data_smithi017_220393_object25580:head v 267'9132, mlcod=267'9127)", 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:37:11.864963+0000", 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.0990120169999997, 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051127207000000001, 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864963+0000", 2024-09-13T14:37:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864963+0000", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864964+0000", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8100000000000002e-07 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864970+0000", 2024-09-13T14:37:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8429999999999998e-06 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864984+0000", 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3725e-05 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.864990+0000", 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4999999999999996e-06 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.874304+0000", 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0093139069999999997 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.874333+0000", 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9206000000000001e-05 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.916009+0000", 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041675605999999997 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:37:11.916091+0000", 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1638999999999995e-05 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:17.074 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:37:17.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:17.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:17.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:17.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:17.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:17.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:17.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:17.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:17.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:17.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:17.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:17] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:37:17.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:17.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:17.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:17.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:17.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:17.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:18.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:18.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:18.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:18.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:18.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:18.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:18.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:18.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:18.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:18.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:18.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:18.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:18.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:18.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:18.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:18.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:18.867 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:18 smithi017 ceph-mon[103001]: pgmap v761: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:37:18.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:18 smithi017 ceph-mon[107556]: pgmap v761: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:37:18.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:37:18.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:19.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:18 smithi189 ceph-mon[90992]: pgmap v761: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-09-13T14:37:19.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:19.130 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:19.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:19.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:19.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:19.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:19.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:19.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:19.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:19.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:19.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:19.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:19.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:19.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:19.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:19.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:20.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:20.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:20.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:20.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:20.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:20.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:20.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:20.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:20.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:20.703 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:20.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:20.707 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:20.710 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:20.809 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:20.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:20.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:20 smithi017 ceph-mon[103001]: pgmap v762: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-13T14:37:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:20 smithi017 ceph-mon[107556]: pgmap v762: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-13T14:37:20.976 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:37:20.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:20.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:20.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:20.982 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:37:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:20 smithi189 ceph-mon[90992]: pgmap v762: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-13T14:37:21.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:21.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:21.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:21.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:21.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:21.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:21.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:21.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:21.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:21.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:21.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:21.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:21.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:21.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:21.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:22.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:22.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:22.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:22.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:22.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:22.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:22.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:22.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:22.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:22.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:22.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:22.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:22.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:22.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:22.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:22.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:22.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:22.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:22.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:22 smithi017 ceph-mon[103001]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:22.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:22 smithi017 ceph-mon[103001]: pgmap v763: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:22.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:22 smithi017 ceph-mon[107556]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:22.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:22 smithi017 ceph-mon[107556]: pgmap v763: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:22.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:22 smithi189 ceph-mon[90992]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:37:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:22 smithi189 ceph-mon[90992]: pgmap v763: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:23.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:23.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:23.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:23.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:23.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:23.389 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":762,"stamp":"2024-09-13T14:37:19.064674+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5849683941,"num_objects":89215,"num_object_clones":0,"num_object_copies":267645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89215,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":149984,"num_write_kb":7637008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130409,"ondisk_log_size":130409,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20077940,"kb_used_data":17280432,"kb_used_omap":36,"kb_used_meta":2797083,"kb_avail":729883276,"statfs":{"total":767960285184,"available":747400474624,"internally_reserved":0,"allocated":17695162368,"data_stored":17684664567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37497,"internal_metadata":2864213383},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2,7,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-537264128,"num_objects":-8198,"num_object_clones":0,"num_object_copies":-24594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8198,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002731"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":414,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.549123+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:37:11.549123+0000","last_peered":"2024-09-13T14:37:11.549123+0000","last_clean":"2024-09-13T14:37:11.549123+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:37:11.549123+0000","last_undegraded":"2024-09-13T14:37:11.549123+0000","last_fullsized":"2024-09-13T14:37:11.549123+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":233,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371011+0000","last_change":"2024-09-13T14:23:27.722537+0000","last_active":"2024-09-13T14:36:58.371011+0000","last_peered":"2024-09-13T14:36:58.371011+0000","last_clean":"2024-09-13T14:36:58.371011+0000","last_became_active":"2024-09-13T14:23:27.722145+0000","last_became_peered":"2024-09-13T14:23:27.722145+0000","last_unstale":"2024-09-13T14:36:58.371011+0000","last_undegraded":"2024-09-13T14:36:58.371011+0000","last_fullsized":"2024-09-13T14:36:58.371011+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:59:47.608210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375930+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:36:58.375930+0000","last_peered":"2024-09-13T14:36:58.375930+0000","last_clean":"2024-09-13T14:36:58.375930+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:36:58.375930+0000","last_undegraded":"2024-09-13T14:36:58.375930+0000","last_fullsized":"2024-09-13T14:36:58.375930+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556008+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:37:11.556008+0000","last_peered":"2024-09-13T14:37:11.556008+0000","last_clean":"2024-09-13T14:37:11.556008+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:37:11.556008+0000","last_undegraded":"2024-09-13T14:37:11.556008+0000","last_fullsized":"2024-09-13T14:37:11.556008+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556303+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:37:11.556303+0000","last_peered":"2024-09-13T14:37:11.556303+0000","last_clean":"2024-09-13T14:37:11.556303+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:37:11.556303+0000","last_undegraded":"2024-09-13T14:37:11.556303+0000","last_fullsized":"2024-09-13T14:37:11.556303+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.370891+0000","last_change":"2024-09-13T14:13:41.127016+0000","last_active":"2024-09-13T14:36:58.370891+0000","last_peered":"2024-09-13T14:36:58.370891+0000","last_clean":"2024-09-13T14:36:58.370891+0000","last_became_active":"2024-09-13T14:13:41.126728+0000","last_became_peered":"2024-09-13T14:13:41.126728+0000","last_unstale":"2024-09-13T14:36:58.370891+0000","last_undegraded":"2024-09-13T14:36:58.370891+0000","last_fullsized":"2024-09-13T14:36:58.370891+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:40.364018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":414,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.551837+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:37:11.551837+0000","last_peered":"2024-09-13T14:37:11.551837+0000","last_clean":"2024-09-13T14:37:11.551837+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:37:11.551837+0000","last_undegraded":"2024-09-13T14:37:11.551837+0000","last_fullsized":"2024-09-13T14:37:11.551837+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":239,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.366945+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:36:58.366945+0000","last_peered":"2024-09-13T14:36:58.366945+0000","last_clean":"2024-09-13T14:36:58.366945+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:36:58.366945+0000","last_undegraded":"2024-09-13T14:36:58.366945+0000","last_fullsized":"2024-09-13T14:36:58.366945+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"267'9225","reported_seq":9277,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:14.678905+0000","last_change":"2024-09-13T14:34:57.524042+0000","last_active":"2024-09-13T14:37:14.678905+0000","last_peered":"2024-09-13T14:37:14.678905+0000","last_clean":"2024-09-13T14:37:14.678905+0000","last_became_active":"2024-09-13T14:34:57.523748+0000","last_became_peered":"2024-09-13T14:34:57.523748+0000","last_unstale":"2024-09-13T14:37:14.678905+0000","last_undegraded":"2024-09-13T14:37:14.678905+0000","last_fullsized":"2024-09-13T14:37:14.678905+0000","mapping_epoch":247,"log_start":"252'1500","ondisk_log_start":"252'1500","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:42.905684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372310016,"num_objects":5681,"num_object_clones":0,"num_object_copies":17043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9225,"num_write_kb":476992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.373657+0000","last_change":"2024-09-13T14:21:39.300857+0000","last_active":"2024-09-13T14:36:58.373657+0000","last_peered":"2024-09-13T14:36:58.373657+0000","last_clean":"2024-09-13T14:36:58.373657+0000","last_became_active":"2024-09-13T14:21:39.300159+0000","last_became_peered":"2024-09-13T14:21:39.300159+0000","last_unstale":"2024-09-13T14:36:58.373657+0000","last_undegraded":"2024-09-13T14:36:58.373657+0000","last_fullsized":"2024-09-13T14:36:58.373657+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:56.724282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":213,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375600+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:36:58.375600+0000","last_peered":"2024-09-13T14:36:58.375600+0000","last_clean":"2024-09-13T14:36:58.375600+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:36:58.375600+0000","last_undegraded":"2024-09-13T14:36:58.375600+0000","last_fullsized":"2024-09-13T14:36:58.375600+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"267'9287","reported_seq":9339,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:14.958746+0000","last_change":"2024-09-13T14:34:57.522300+0000","last_active":"2024-09-13T14:37:14.958746+0000","last_peered":"2024-09-13T14:37:14.958746+0000","last_clean":"2024-09-13T14:37:14.958746+0000","last_became_active":"2024-09-13T14:34:57.522005+0000","last_became_peered":"2024-09-13T14:34:57.522005+0000","last_unstale":"2024-09-13T14:37:14.958746+0000","last_undegraded":"2024-09-13T14:37:14.958746+0000","last_fullsized":"2024-09-13T14:37:14.958746+0000","mapping_epoch":247,"log_start":"254'2600","ondisk_log_start":"254'2600","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:56.018499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378339328,"num_objects":5773,"num_object_clones":0,"num_object_copies":17319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9287,"num_write_kb":481920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":250,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371100+0000","last_change":"2024-09-13T14:34:46.649053+0000","last_active":"2024-09-13T14:36:58.371100+0000","last_peered":"2024-09-13T14:36:58.371100+0000","last_clean":"2024-09-13T14:36:58.371100+0000","last_became_active":"2024-09-13T14:34:46.648725+0000","last_became_peered":"2024-09-13T14:34:46.648725+0000","last_unstale":"2024-09-13T14:36:58.371100+0000","last_undegraded":"2024-09-13T14:36:58.371100+0000","last_fullsized":"2024-09-13T14:36:58.371100+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:02:03.935325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369477+0000","last_change":"2024-09-13T14:34:46.594184+0000","last_active":"2024-09-13T14:36:58.369477+0000","last_peered":"2024-09-13T14:36:58.369477+0000","last_clean":"2024-09-13T14:36:58.369477+0000","last_became_active":"2024-09-13T14:34:46.593911+0000","last_became_peered":"2024-09-13T14:34:46.593911+0000","last_unstale":"2024-09-13T14:36:58.369477+0000","last_undegraded":"2024-09-13T14:36:58.369477+0000","last_fullsized":"2024-09-13T14:36:58.369477+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:41.568032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"267'9371","reported_seq":9423,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.153703+0000","last_change":"2024-09-13T14:34:57.526403+0000","last_active":"2024-09-13T14:37:18.153703+0000","last_peered":"2024-09-13T14:37:18.153703+0000","last_clean":"2024-09-13T14:37:18.153703+0000","last_became_active":"2024-09-13T14:34:57.525671+0000","last_became_peered":"2024-09-13T14:34:57.525671+0000","last_unstale":"2024-09-13T14:37:18.153703+0000","last_undegraded":"2024-09-13T14:37:18.153703+0000","last_fullsized":"2024-09-13T14:37:18.153703+0000","mapping_epoch":247,"log_start":"252'1200","ondisk_log_start":"252'1200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:22.342551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361693184,"num_objects":5519,"num_object_clones":0,"num_object_copies":16557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9371,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":215,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371394+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:36:58.371394+0000","last_peered":"2024-09-13T14:36:58.371394+0000","last_clean":"2024-09-13T14:36:58.371394+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:36:58.371394+0000","last_undegraded":"2024-09-13T14:36:58.371394+0000","last_fullsized":"2024-09-13T14:36:58.371394+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.367284+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:36:58.367284+0000","last_peered":"2024-09-13T14:36:58.367284+0000","last_clean":"2024-09-13T14:36:58.367284+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:36:58.367284+0000","last_undegraded":"2024-09-13T14:36:58.367284+0000","last_fullsized":"2024-09-13T14:36:58.367284+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"267'9451","reported_seq":9503,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:16.658332+0000","last_change":"2024-09-13T14:34:57.521543+0000","last_active":"2024-09-13T14:37:16.658332+0000","last_peered":"2024-09-13T14:37:16.658332+0000","last_clean":"2024-09-13T14:37:16.658332+0000","last_became_active":"2024-09-13T14:34:57.521269+0000","last_became_peered":"2024-09-13T14:34:57.521269+0000","last_unstale":"2024-09-13T14:37:16.658332+0000","last_undegraded":"2024-09-13T14:37:16.658332+0000","last_fullsized":"2024-09-13T14:37:16.658332+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":9451,"log_dups_size":0,"ondisk_log_size":9451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:21.067198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9451,"num_write_kb":484800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375982+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:36:58.375982+0000","last_peered":"2024-09-13T14:36:58.375982+0000","last_clean":"2024-09-13T14:36:58.375982+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:36:58.375982+0000","last_undegraded":"2024-09-13T14:36:58.375982+0000","last_fullsized":"2024-09-13T14:36:58.375982+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":411,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556513+0000","last_change":"2024-09-13T14:34:46.616314+0000","last_active":"2024-09-13T14:37:11.556513+0000","last_peered":"2024-09-13T14:37:11.556513+0000","last_clean":"2024-09-13T14:37:11.556513+0000","last_became_active":"2024-09-13T14:34:46.611776+0000","last_became_peered":"2024-09-13T14:34:46.611776+0000","last_unstale":"2024-09-13T14:37:11.556513+0000","last_undegraded":"2024-09-13T14:37:11.556513+0000","last_fullsized":"2024-09-13T14:37:11.556513+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:44.754447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"267'9339","reported_seq":9391,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.084567+0000","last_change":"2024-09-13T14:34:57.526751+0000","last_active":"2024-09-13T14:37:18.084567+0000","last_peered":"2024-09-13T14:37:18.084567+0000","last_clean":"2024-09-13T14:37:18.084567+0000","last_became_active":"2024-09-13T14:34:57.526442+0000","last_became_peered":"2024-09-13T14:34:57.526442+0000","last_unstale":"2024-09-13T14:37:18.084567+0000","last_undegraded":"2024-09-13T14:37:18.084567+0000","last_fullsized":"2024-09-13T14:37:18.084567+0000","mapping_epoch":247,"log_start":"252'1200","ondisk_log_start":"252'1200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:09.948770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357498880,"num_objects":5455,"num_object_clones":0,"num_object_copies":16365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9339,"num_write_kb":473408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":378,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.370752+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:36:58.370752+0000","last_peered":"2024-09-13T14:36:58.370752+0000","last_clean":"2024-09-13T14:36:58.370752+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:36:58.370752+0000","last_undegraded":"2024-09-13T14:36:58.370752+0000","last_fullsized":"2024-09-13T14:36:58.370752+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":401,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545609+0000","last_change":"2024-09-13T14:24:00.658508+0000","last_active":"2024-09-13T14:37:11.545609+0000","last_peered":"2024-09-13T14:37:11.545609+0000","last_clean":"2024-09-13T14:37:11.545609+0000","last_became_active":"2024-09-13T14:24:00.657990+0000","last_became_peered":"2024-09-13T14:24:00.657990+0000","last_unstale":"2024-09-13T14:37:11.545609+0000","last_undegraded":"2024-09-13T14:37:11.545609+0000","last_fullsized":"2024-09-13T14:37:11.545609+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:26.468649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":231,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369544+0000","last_change":"2024-09-13T14:24:00.653246+0000","last_active":"2024-09-13T14:36:58.369544+0000","last_peered":"2024-09-13T14:36:58.369544+0000","last_clean":"2024-09-13T14:36:58.369544+0000","last_became_active":"2024-09-13T14:24:00.652395+0000","last_became_peered":"2024-09-13T14:24:00.652395+0000","last_unstale":"2024-09-13T14:36:58.369544+0000","last_undegraded":"2024-09-13T14:36:58.369544+0000","last_fullsized":"2024-09-13T14:36:58.369544+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:41.772959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"267'9336","reported_seq":9386,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:16.628344+0000","last_change":"2024-09-13T14:34:57.525034+0000","last_active":"2024-09-13T14:37:16.628344+0000","last_peered":"2024-09-13T14:37:16.628344+0000","last_clean":"2024-09-13T14:37:16.628344+0000","last_became_active":"2024-09-13T14:34:57.524740+0000","last_became_peered":"2024-09-13T14:34:57.524740+0000","last_unstale":"2024-09-13T14:37:16.628344+0000","last_undegraded":"2024-09-13T14:37:16.628344+0000","last_fullsized":"2024-09-13T14:37:16.628344+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":9336,"log_dups_size":0,"ondisk_log_size":9336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:44.966638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366084096,"num_objects":5586,"num_object_clones":0,"num_object_copies":16758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9336,"num_write_kb":477504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":125,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375551+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:36:58.375551+0000","last_peered":"2024-09-13T14:36:58.375551+0000","last_clean":"2024-09-13T14:36:58.375551+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:36:58.375551+0000","last_undegraded":"2024-09-13T14:36:58.375551+0000","last_fullsized":"2024-09-13T14:36:58.375551+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.367117+0000","last_change":"2024-09-13T14:34:46.575576+0000","last_active":"2024-09-13T14:36:58.367117+0000","last_peered":"2024-09-13T14:36:58.367117+0000","last_clean":"2024-09-13T14:36:58.367117+0000","last_became_active":"2024-09-13T14:34:46.575291+0000","last_became_peered":"2024-09-13T14:34:46.575291+0000","last_unstale":"2024-09-13T14:36:58.367117+0000","last_undegraded":"2024-09-13T14:36:58.367117+0000","last_fullsized":"2024-09-13T14:36:58.367117+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:30.242110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":241,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369610+0000","last_change":"2024-09-13T14:23:27.721563+0000","last_active":"2024-09-13T14:36:58.369610+0000","last_peered":"2024-09-13T14:36:58.369610+0000","last_clean":"2024-09-13T14:36:58.369610+0000","last_became_active":"2024-09-13T14:23:27.721263+0000","last_became_peered":"2024-09-13T14:23:27.721263+0000","last_unstale":"2024-09-13T14:36:58.369610+0000","last_undegraded":"2024-09-13T14:36:58.369610+0000","last_fullsized":"2024-09-13T14:36:58.369610+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:07.923956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"267'9372","reported_seq":9401,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:17.887614+0000","last_change":"2024-09-13T14:34:57.522071+0000","last_active":"2024-09-13T14:37:17.887614+0000","last_peered":"2024-09-13T14:37:17.887614+0000","last_clean":"2024-09-13T14:37:17.887614+0000","last_became_active":"2024-09-13T14:34:57.521798+0000","last_became_peered":"2024-09-13T14:34:57.521798+0000","last_unstale":"2024-09-13T14:37:17.887614+0000","last_undegraded":"2024-09-13T14:37:17.887614+0000","last_fullsized":"2024-09-13T14:37:17.887614+0000","mapping_epoch":247,"log_start":"252'1200","ondisk_log_start":"252'1200","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:47.723539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902168,"num_objects":5630,"num_object_clones":0,"num_object_copies":16890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5630,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9372,"num_write_kb":480001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369547+0000","last_change":"2024-09-13T14:23:27.720849+0000","last_active":"2024-09-13T14:36:58.369547+0000","last_peered":"2024-09-13T14:36:58.369547+0000","last_clean":"2024-09-13T14:36:58.369547+0000","last_became_active":"2024-09-13T14:23:27.720550+0000","last_became_peered":"2024-09-13T14:23:27.720550+0000","last_unstale":"2024-09-13T14:36:58.369547+0000","last_undegraded":"2024-09-13T14:36:58.369547+0000","last_fullsized":"2024-09-13T14:36:58.369547+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:52.283708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369249+0000","last_change":"2024-09-13T14:34:46.648302+0000","last_active":"2024-09-13T14:36:58.369249+0000","last_peered":"2024-09-13T14:36:58.369249+0000","last_clean":"2024-09-13T14:36:58.369249+0000","last_became_active":"2024-09-13T14:34:46.647761+0000","last_became_peered":"2024-09-13T14:34:46.647761+0000","last_unstale":"2024-09-13T14:36:58.369249+0000","last_undegraded":"2024-09-13T14:36:58.369249+0000","last_fullsized":"2024-09-13T14:36:58.369249+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:22.055371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"267'9106","reported_seq":9134,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:17.973738+0000","last_change":"2024-09-13T14:34:57.521324+0000","last_active":"2024-09-13T14:37:17.973738+0000","last_peered":"2024-09-13T14:37:17.973738+0000","last_clean":"2024-09-13T14:37:17.973738+0000","last_became_active":"2024-09-13T14:34:57.520992+0000","last_became_peered":"2024-09-13T14:34:57.520992+0000","last_unstale":"2024-09-13T14:37:17.973738+0000","last_undegraded":"2024-09-13T14:37:17.973738+0000","last_fullsized":"2024-09-13T14:37:17.973738+0000","mapping_epoch":247,"log_start":"252'900","ondisk_log_start":"252'900","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.105872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349044736,"num_objects":5326,"num_object_clones":0,"num_object_copies":15978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9106,"num_write_kb":461824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":118,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375709+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:36:58.375709+0000","last_peered":"2024-09-13T14:36:58.375709+0000","last_clean":"2024-09-13T14:36:58.375709+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:36:58.375709+0000","last_undegraded":"2024-09-13T14:36:58.375709+0000","last_fullsized":"2024-09-13T14:36:58.375709+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":416,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545823+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:37:11.545823+0000","last_peered":"2024-09-13T14:37:11.545823+0000","last_clean":"2024-09-13T14:37:11.545823+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:37:11.545823+0000","last_undegraded":"2024-09-13T14:37:11.545823+0000","last_fullsized":"2024-09-13T14:37:11.545823+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":364,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.546974+0000","last_change":"2024-09-13T14:34:46.624474+0000","last_active":"2024-09-13T14:37:11.546974+0000","last_peered":"2024-09-13T14:37:11.546974+0000","last_clean":"2024-09-13T14:37:11.546974+0000","last_became_active":"2024-09-13T14:34:46.624138+0000","last_became_peered":"2024-09-13T14:34:46.624138+0000","last_unstale":"2024-09-13T14:37:11.546974+0000","last_undegraded":"2024-09-13T14:37:11.546974+0000","last_fullsized":"2024-09-13T14:37:11.546974+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:51.252508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"267'9253","reported_seq":9305,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.150711+0000","last_change":"2024-09-13T14:34:57.525058+0000","last_active":"2024-09-13T14:37:18.150711+0000","last_peered":"2024-09-13T14:37:18.150711+0000","last_clean":"2024-09-13T14:37:18.150711+0000","last_became_active":"2024-09-13T14:34:57.524514+0000","last_became_peered":"2024-09-13T14:34:57.524514+0000","last_unstale":"2024-09-13T14:37:18.150711+0000","last_undegraded":"2024-09-13T14:37:18.150711+0000","last_fullsized":"2024-09-13T14:37:18.150711+0000","mapping_epoch":247,"log_start":"252'1100","ondisk_log_start":"252'1100","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8153,"log_dups_size":0,"ondisk_log_size":8153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:17.836817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362872832,"num_objects":5537,"num_object_clones":0,"num_object_copies":16611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9253,"num_write_kb":473280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":369,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371195+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:36:58.371195+0000","last_peered":"2024-09-13T14:36:58.371195+0000","last_clean":"2024-09-13T14:36:58.371195+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:36:58.371195+0000","last_undegraded":"2024-09-13T14:36:58.371195+0000","last_fullsized":"2024-09-13T14:36:58.371195+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":832,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375711+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:36:58.375711+0000","last_peered":"2024-09-13T14:36:58.375711+0000","last_clean":"2024-09-13T14:36:58.375711+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:36:58.375711+0000","last_undegraded":"2024-09-13T14:36:58.375711+0000","last_fullsized":"2024-09-13T14:36:58.375711+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":358,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545948+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:37:11.545948+0000","last_peered":"2024-09-13T14:37:11.545948+0000","last_clean":"2024-09-13T14:37:11.545948+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:37:11.545948+0000","last_undegraded":"2024-09-13T14:37:11.545948+0000","last_fullsized":"2024-09-13T14:37:11.545948+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":213,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.368688+0000","last_change":"2024-09-13T14:13:41.147211+0000","last_active":"2024-09-13T14:36:58.368688+0000","last_peered":"2024-09-13T14:36:58.368688+0000","last_clean":"2024-09-13T14:36:58.368688+0000","last_became_active":"2024-09-13T14:13:41.146953+0000","last_became_peered":"2024-09-13T14:13:41.146953+0000","last_unstale":"2024-09-13T14:36:58.368688+0000","last_undegraded":"2024-09-13T14:36:58.368688+0000","last_fullsized":"2024-09-13T14:36:58.368688+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:10.747362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"267'9405","reported_seq":9433,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.082358+0000","last_change":"2024-09-13T14:34:57.525197+0000","last_active":"2024-09-13T14:37:18.082358+0000","last_peered":"2024-09-13T14:37:18.082358+0000","last_clean":"2024-09-13T14:37:18.082358+0000","last_became_active":"2024-09-13T14:34:57.524916+0000","last_became_peered":"2024-09-13T14:34:57.524916+0000","last_unstale":"2024-09-13T14:37:18.082358+0000","last_undegraded":"2024-09-13T14:37:18.082358+0000","last_fullsized":"2024-09-13T14:37:18.082358+0000","mapping_epoch":247,"log_start":"252'1700","ondisk_log_start":"252'1700","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:01.282676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364052480,"num_objects":5555,"num_object_clones":0,"num_object_copies":16665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9405,"num_write_kb":478720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":203,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547117+0000","last_change":"2024-09-13T14:34:47.477679+0000","last_active":"2024-09-13T14:37:11.547117+0000","last_peered":"2024-09-13T14:37:11.547117+0000","last_clean":"2024-09-13T14:37:11.547117+0000","last_became_active":"2024-09-13T14:34:46.634361+0000","last_became_peered":"2024-09-13T14:34:46.634361+0000","last_unstale":"2024-09-13T14:37:11.547117+0000","last_undegraded":"2024-09-13T14:37:11.547117+0000","last_fullsized":"2024-09-13T14:37:11.547117+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:44.197390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":657,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371059+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:36:58.371059+0000","last_peered":"2024-09-13T14:36:58.371059+0000","last_clean":"2024-09-13T14:36:58.371059+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:36:58.371059+0000","last_undegraded":"2024-09-13T14:36:58.371059+0000","last_fullsized":"2024-09-13T14:36:58.371059+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.567150+0000","last_change":"2024-09-13T14:24:00.653688+0000","last_active":"2024-09-13T14:37:11.567150+0000","last_peered":"2024-09-13T14:37:11.567150+0000","last_clean":"2024-09-13T14:37:11.567150+0000","last_became_active":"2024-09-13T14:24:00.653378+0000","last_became_peered":"2024-09-13T14:24:00.653378+0000","last_unstale":"2024-09-13T14:37:11.567150+0000","last_undegraded":"2024-09-13T14:37:11.567150+0000","last_fullsized":"2024-09-13T14:37:11.567150+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:05.918623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":423,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.562726+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:37:11.562726+0000","last_peered":"2024-09-13T14:37:11.562726+0000","last_clean":"2024-09-13T14:37:11.562726+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:37:11.562726+0000","last_undegraded":"2024-09-13T14:37:11.562726+0000","last_fullsized":"2024-09-13T14:37:11.562726+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"267'9414","reported_seq":9466,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.144614+0000","last_change":"2024-09-13T14:34:57.525939+0000","last_active":"2024-09-13T14:37:18.144614+0000","last_peered":"2024-09-13T14:37:18.144614+0000","last_clean":"2024-09-13T14:37:18.144614+0000","last_became_active":"2024-09-13T14:34:57.525621+0000","last_became_peered":"2024-09-13T14:34:57.525621+0000","last_unstale":"2024-09-13T14:37:18.144614+0000","last_undegraded":"2024-09-13T14:37:18.144614+0000","last_fullsized":"2024-09-13T14:37:18.144614+0000","mapping_epoch":247,"log_start":"252'1300","ondisk_log_start":"252'1300","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8114,"log_dups_size":0,"ondisk_log_size":8114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:26.471406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360185856,"num_objects":5496,"num_object_clones":0,"num_object_copies":16488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9414,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":378,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371166+0000","last_change":"2024-09-13T14:30:37.361996+0000","last_active":"2024-09-13T14:36:58.371166+0000","last_peered":"2024-09-13T14:36:58.371166+0000","last_clean":"2024-09-13T14:36:58.371166+0000","last_became_active":"2024-09-13T14:30:11.437181+0000","last_became_peered":"2024-09-13T14:30:11.437181+0000","last_unstale":"2024-09-13T14:36:58.371166+0000","last_undegraded":"2024-09-13T14:36:58.371166+0000","last_fullsized":"2024-09-13T14:36:58.371166+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:11.739749+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":829,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547062+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:37:11.547062+0000","last_peered":"2024-09-13T14:37:11.547062+0000","last_clean":"2024-09-13T14:37:11.547062+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:37:11.547062+0000","last_undegraded":"2024-09-13T14:37:11.547062+0000","last_fullsized":"2024-09-13T14:37:11.547062+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":355,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.549700+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:37:11.549700+0000","last_peered":"2024-09-13T14:37:11.549700+0000","last_clean":"2024-09-13T14:37:11.549700+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:37:11.549700+0000","last_undegraded":"2024-09-13T14:37:11.549700+0000","last_fullsized":"2024-09-13T14:37:11.549700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":406,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547507+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:37:11.547507+0000","last_peered":"2024-09-13T14:37:11.547507+0000","last_clean":"2024-09-13T14:37:11.547507+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:37:11.547507+0000","last_undegraded":"2024-09-13T14:37:11.547507+0000","last_fullsized":"2024-09-13T14:37:11.547507+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"267'9495","reported_seq":9522,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:17.959693+0000","last_change":"2024-09-13T14:34:57.522448+0000","last_active":"2024-09-13T14:37:17.959693+0000","last_peered":"2024-09-13T14:37:17.959693+0000","last_clean":"2024-09-13T14:37:17.959693+0000","last_became_active":"2024-09-13T14:34:57.522167+0000","last_became_peered":"2024-09-13T14:34:57.522167+0000","last_unstale":"2024-09-13T14:37:17.959693+0000","last_undegraded":"2024-09-13T14:37:17.959693+0000","last_fullsized":"2024-09-13T14:37:17.959693+0000","mapping_epoch":247,"log_start":"252'1300","ondisk_log_start":"252'1300","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8195,"log_dups_size":0,"ondisk_log_size":8195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:22:47.802147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364183552,"num_objects":5557,"num_object_clones":0,"num_object_copies":16671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9495,"num_write_kb":481664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":118,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375586+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:36:58.375586+0000","last_peered":"2024-09-13T14:36:58.375586+0000","last_clean":"2024-09-13T14:36:58.375586+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:36:58.375586+0000","last_undegraded":"2024-09-13T14:36:58.375586+0000","last_fullsized":"2024-09-13T14:36:58.375586+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":898,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.378579+0000","last_change":"2024-09-13T14:34:46.566197+0000","last_active":"2024-09-13T14:36:58.378579+0000","last_peered":"2024-09-13T14:36:58.378579+0000","last_clean":"2024-09-13T14:36:58.378579+0000","last_became_active":"2024-09-13T14:34:46.565768+0000","last_became_peered":"2024-09-13T14:34:46.565768+0000","last_unstale":"2024-09-13T14:36:58.378579+0000","last_undegraded":"2024-09-13T14:36:58.378579+0000","last_fullsized":"2024-09-13T14:36:58.378579+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:12:46.443713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369404+0000","last_change":"2024-09-13T14:21:39.300373+0000","last_active":"2024-09-13T14:36:58.369404+0000","last_peered":"2024-09-13T14:36:58.369404+0000","last_clean":"2024-09-13T14:36:58.369404+0000","last_became_active":"2024-09-13T14:21:39.300069+0000","last_became_peered":"2024-09-13T14:21:39.300069+0000","last_unstale":"2024-09-13T14:36:58.369404+0000","last_undegraded":"2024-09-13T14:36:58.369404+0000","last_fullsized":"2024-09-13T14:36:58.369404+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:46.500594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":376,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547187+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:37:11.547187+0000","last_peered":"2024-09-13T14:37:11.547187+0000","last_clean":"2024-09-13T14:37:11.547187+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:37:11.547187+0000","last_undegraded":"2024-09-13T14:37:11.547187+0000","last_fullsized":"2024-09-13T14:37:11.547187+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"267'9375","reported_seq":9401,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:18.101588+0000","last_change":"2024-09-13T14:34:57.521524+0000","last_active":"2024-09-13T14:37:18.101588+0000","last_peered":"2024-09-13T14:37:18.101588+0000","last_clean":"2024-09-13T14:37:18.101588+0000","last_became_active":"2024-09-13T14:34:57.520850+0000","last_became_peered":"2024-09-13T14:34:57.520850+0000","last_unstale":"2024-09-13T14:37:18.101588+0000","last_undegraded":"2024-09-13T14:37:18.101588+0000","last_fullsized":"2024-09-13T14:37:18.101588+0000","mapping_epoch":247,"log_start":"253'2000","ondisk_log_start":"253'2000","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:13:42.515897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360513536,"num_objects":5501,"num_object_clones":0,"num_object_copies":16503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9375,"num_write_kb":476032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369069+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:36:58.369069+0000","last_peered":"2024-09-13T14:36:58.369069+0000","last_clean":"2024-09-13T14:36:58.369069+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:36:58.369069+0000","last_undegraded":"2024-09-13T14:36:58.369069+0000","last_fullsized":"2024-09-13T14:36:58.369069+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.376124+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:36:58.376124+0000","last_peered":"2024-09-13T14:36:58.376124+0000","last_clean":"2024-09-13T14:36:58.376124+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:36:58.376124+0000","last_undegraded":"2024-09-13T14:36:58.376124+0000","last_fullsized":"2024-09-13T14:36:58.376124+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":955,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547359+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:37:11.547359+0000","last_peered":"2024-09-13T14:37:11.547359+0000","last_clean":"2024-09-13T14:37:11.547359+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:37:11.547359+0000","last_undegraded":"2024-09-13T14:37:11.547359+0000","last_fullsized":"2024-09-13T14:37:11.547359+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":334,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.546184+0000","last_change":"2024-09-13T14:34:46.616754+0000","last_active":"2024-09-13T14:37:11.546184+0000","last_peered":"2024-09-13T14:37:11.546184+0000","last_clean":"2024-09-13T14:37:11.546184+0000","last_became_active":"2024-09-13T14:34:46.616476+0000","last_became_peered":"2024-09-13T14:34:46.616476+0000","last_unstale":"2024-09-13T14:37:11.546184+0000","last_undegraded":"2024-09-13T14:37:11.546184+0000","last_fullsized":"2024-09-13T14:37:11.546184+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:14.670350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"267'9481","reported_seq":9509,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:17.982175+0000","last_change":"2024-09-13T14:34:57.527395+0000","last_active":"2024-09-13T14:37:17.982175+0000","last_peered":"2024-09-13T14:37:17.982175+0000","last_clean":"2024-09-13T14:37:17.982175+0000","last_became_active":"2024-09-13T14:34:57.527133+0000","last_became_peered":"2024-09-13T14:34:57.527133+0000","last_unstale":"2024-09-13T14:37:17.982175+0000","last_undegraded":"2024-09-13T14:37:17.982175+0000","last_fullsized":"2024-09-13T14:37:17.982175+0000","mapping_epoch":247,"log_start":"0'0","ondisk_log_start":"0'0","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":9481,"log_dups_size":0,"ondisk_log_size":9481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:16.255103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355794944,"num_objects":5429,"num_object_clones":0,"num_object_copies":16287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9481,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.368542+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:36:58.368542+0000","last_peered":"2024-09-13T14:36:58.368542+0000","last_clean":"2024-09-13T14:36:58.368542+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:36:58.368542+0000","last_undegraded":"2024-09-13T14:36:58.368542+0000","last_fullsized":"2024-09-13T14:36:58.368542+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":355,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547268+0000","last_change":"2024-09-13T14:34:46.583932+0000","last_active":"2024-09-13T14:37:11.547268+0000","last_peered":"2024-09-13T14:37:11.547268+0000","last_clean":"2024-09-13T14:37:11.547268+0000","last_became_active":"2024-09-13T14:34:46.580751+0000","last_became_peered":"2024-09-13T14:34:46.580751+0000","last_unstale":"2024-09-13T14:37:11.547268+0000","last_undegraded":"2024-09-13T14:37:11.547268+0000","last_fullsized":"2024-09-13T14:37:11.547268+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:38.905892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":577,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371462+0000","last_change":"2024-09-13T14:21:39.297028+0000","last_active":"2024-09-13T14:36:58.371462+0000","last_peered":"2024-09-13T14:36:58.371462+0000","last_clean":"2024-09-13T14:36:58.371462+0000","last_became_active":"2024-09-13T14:21:39.296535+0000","last_became_peered":"2024-09-13T14:21:39.296535+0000","last_unstale":"2024-09-13T14:36:58.371462+0000","last_undegraded":"2024-09-13T14:36:58.371462+0000","last_fullsized":"2024-09-13T14:36:58.371462+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:52.615201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":390,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556418+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:37:11.556418+0000","last_peered":"2024-09-13T14:37:11.556418+0000","last_clean":"2024-09-13T14:37:11.556418+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:37:11.556418+0000","last_undegraded":"2024-09-13T14:37:11.556418+0000","last_fullsized":"2024-09-13T14:37:11.556418+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"267'8987","reported_seq":9039,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:14.899409+0000","last_change":"2024-09-13T14:34:57.521462+0000","last_active":"2024-09-13T14:37:14.899409+0000","last_peered":"2024-09-13T14:37:14.899409+0000","last_clean":"2024-09-13T14:37:14.899409+0000","last_became_active":"2024-09-13T14:34:57.521178+0000","last_became_peered":"2024-09-13T14:34:57.521178+0000","last_unstale":"2024-09-13T14:37:14.899409+0000","last_undegraded":"2024-09-13T14:37:14.899409+0000","last_fullsized":"2024-09-13T14:37:14.899409+0000","mapping_epoch":247,"log_start":"253'2300","ondisk_log_start":"253'2300","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:32.574385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372834304,"num_objects":5689,"num_object_clones":0,"num_object_copies":17067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8987,"num_write_kb":469632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":118,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369379+0000","last_change":"2024-09-13T14:30:37.305657+0000","last_active":"2024-09-13T14:36:58.369379+0000","last_peered":"2024-09-13T14:36:58.369379+0000","last_clean":"2024-09-13T14:36:58.369379+0000","last_became_active":"2024-09-13T14:30:11.459960+0000","last_became_peered":"2024-09-13T14:30:11.459960+0000","last_unstale":"2024-09-13T14:36:58.369379+0000","last_undegraded":"2024-09-13T14:36:58.369379+0000","last_fullsized":"2024-09-13T14:36:58.369379+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.305551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.305551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:10:31.136890+0000","scrub_duration":0.0028776439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":402,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.552043+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:37:11.552043+0000","last_peered":"2024-09-13T14:37:11.552043+0000","last_clean":"2024-09-13T14:37:11.552043+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:37:11.552043+0000","last_undegraded":"2024-09-13T14:37:11.552043+0000","last_fullsized":"2024-09-13T14:37:11.552043+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":689,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.546702+0000","last_change":"2024-09-13T14:34:46.652006+0000","last_active":"2024-09-13T14:37:11.546702+0000","last_peered":"2024-09-13T14:37:11.546702+0000","last_clean":"2024-09-13T14:37:11.546702+0000","last_became_active":"2024-09-13T14:34:46.651532+0000","last_became_peered":"2024-09-13T14:34:46.651532+0000","last_unstale":"2024-09-13T14:37:11.546702+0000","last_undegraded":"2024-09-13T14:37:11.546702+0000","last_fullsized":"2024-09-13T14:37:11.546702+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:35.907330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545624+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:37:11.545624+0000","last_peered":"2024-09-13T14:37:11.545624+0000","last_clean":"2024-09-13T14:37:11.545624+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:37:11.545624+0000","last_undegraded":"2024-09-13T14:37:11.545624+0000","last_fullsized":"2024-09-13T14:37:11.545624+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"267'9180","reported_seq":9232,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:14.973998+0000","last_change":"2024-09-13T14:34:57.522594+0000","last_active":"2024-09-13T14:37:14.973998+0000","last_peered":"2024-09-13T14:37:14.973998+0000","last_clean":"2024-09-13T14:37:14.973998+0000","last_became_active":"2024-09-13T14:34:57.520039+0000","last_became_peered":"2024-09-13T14:34:57.520039+0000","last_unstale":"2024-09-13T14:37:14.973998+0000","last_undegraded":"2024-09-13T14:37:14.973998+0000","last_fullsized":"2024-09-13T14:37:14.973998+0000","mapping_epoch":247,"log_start":"253'2500","ondisk_log_start":"253'2500","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:37.564876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377094144,"num_objects":5754,"num_object_clones":0,"num_object_copies":17262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9180,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":240,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.548974+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:37:11.548974+0000","last_peered":"2024-09-13T14:37:11.548974+0000","last_clean":"2024-09-13T14:37:11.548974+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:37:11.548974+0000","last_undegraded":"2024-09-13T14:37:11.548974+0000","last_fullsized":"2024-09-13T14:37:11.548974+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:17.492554+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":415,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556207+0000","last_change":"2024-09-13T14:34:46.577789+0000","last_active":"2024-09-13T14:37:11.556207+0000","last_peered":"2024-09-13T14:37:11.556207+0000","last_clean":"2024-09-13T14:37:11.556207+0000","last_became_active":"2024-09-13T14:34:46.574952+0000","last_became_peered":"2024-09-13T14:34:46.574952+0000","last_unstale":"2024-09-13T14:37:11.556207+0000","last_undegraded":"2024-09-13T14:37:11.556207+0000","last_fullsized":"2024-09-13T14:37:11.556207+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:20:04.720034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":685,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369294+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:36:58.369294+0000","last_peered":"2024-09-13T14:36:58.369294+0000","last_clean":"2024-09-13T14:36:58.369294+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:36:58.369294+0000","last_undegraded":"2024-09-13T14:36:58.369294+0000","last_fullsized":"2024-09-13T14:36:58.369294+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":214,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.368880+0000","last_change":"2024-09-13T14:13:41.142965+0000","last_active":"2024-09-13T14:36:58.368880+0000","last_peered":"2024-09-13T14:36:58.368880+0000","last_clean":"2024-09-13T14:36:58.368880+0000","last_became_active":"2024-09-13T14:13:41.142700+0000","last_became_peered":"2024-09-13T14:13:41.142700+0000","last_unstale":"2024-09-13T14:36:58.368880+0000","last_undegraded":"2024-09-13T14:36:58.368880+0000","last_fullsized":"2024-09-13T14:36:58.368880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:45.944820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369272+0000","last_change":"2024-09-13T14:34:46.645586+0000","last_active":"2024-09-13T14:36:58.369272+0000","last_peered":"2024-09-13T14:36:58.369272+0000","last_clean":"2024-09-13T14:36:58.369272+0000","last_became_active":"2024-09-13T14:34:46.645294+0000","last_became_peered":"2024-09-13T14:34:46.645294+0000","last_unstale":"2024-09-13T14:36:58.369272+0000","last_undegraded":"2024-09-13T14:36:58.369272+0000","last_fullsized":"2024-09-13T14:36:58.369272+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:55.643942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":237,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369588+0000","last_change":"2024-09-13T14:13:41.127091+0000","last_active":"2024-09-13T14:36:58.369588+0000","last_peered":"2024-09-13T14:36:58.369588+0000","last_clean":"2024-09-13T14:36:58.369588+0000","last_became_active":"2024-09-13T14:13:41.126709+0000","last_became_peered":"2024-09-13T14:13:41.126709+0000","last_unstale":"2024-09-13T14:36:58.369588+0000","last_undegraded":"2024-09-13T14:36:58.369588+0000","last_fullsized":"2024-09-13T14:36:58.369588+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:58.979364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":354,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.546035+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:37:11.546035+0000","last_peered":"2024-09-13T14:37:11.546035+0000","last_clean":"2024-09-13T14:37:11.546035+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:37:11.546035+0000","last_undegraded":"2024-09-13T14:37:11.546035+0000","last_fullsized":"2024-09-13T14:37:11.546035+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":234,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.371257+0000","last_change":"2024-09-13T14:24:00.650848+0000","last_active":"2024-09-13T14:36:58.371257+0000","last_peered":"2024-09-13T14:36:58.371257+0000","last_clean":"2024-09-13T14:36:58.371257+0000","last_became_active":"2024-09-13T14:24:00.650437+0000","last_became_peered":"2024-09-13T14:24:00.650437+0000","last_unstale":"2024-09-13T14:36:58.371257+0000","last_undegraded":"2024-09-13T14:36:58.371257+0000","last_fullsized":"2024-09-13T14:36:58.371257+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:37.132183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.376265+0000","last_change":"2024-09-13T14:34:46.565021+0000","last_active":"2024-09-13T14:36:58.376265+0000","last_peered":"2024-09-13T14:36:58.376265+0000","last_clean":"2024-09-13T14:36:58.376265+0000","last_became_active":"2024-09-13T14:34:46.563784+0000","last_became_peered":"2024-09-13T14:34:46.563784+0000","last_unstale":"2024-09-13T14:36:58.376265+0000","last_undegraded":"2024-09-13T14:36:58.376265+0000","last_fullsized":"2024-09-13T14:36:58.376265+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:00.380107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":358,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.549968+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:37:11.549968+0000","last_peered":"2024-09-13T14:37:11.549968+0000","last_clean":"2024-09-13T14:37:11.549968+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:37:11.549968+0000","last_undegraded":"2024-09-13T14:37:11.549968+0000","last_fullsized":"2024-09-13T14:37:11.549968+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.560501+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:37:11.560501+0000","last_peered":"2024-09-13T14:37:11.560501+0000","last_clean":"2024-09-13T14:37:11.560501+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:37:11.560501+0000","last_undegraded":"2024-09-13T14:37:11.560501+0000","last_fullsized":"2024-09-13T14:37:11.560501+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":414,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545517+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:37:11.545517+0000","last_peered":"2024-09-13T14:37:11.545517+0000","last_clean":"2024-09-13T14:37:11.545517+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:37:11.545517+0000","last_undegraded":"2024-09-13T14:37:11.545517+0000","last_fullsized":"2024-09-13T14:37:11.545517+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369225+0000","last_change":"2024-09-13T14:34:46.570197+0000","last_active":"2024-09-13T14:36:58.369225+0000","last_peered":"2024-09-13T14:36:58.369225+0000","last_clean":"2024-09-13T14:36:58.369225+0000","last_became_active":"2024-09-13T14:34:46.568152+0000","last_became_peered":"2024-09-13T14:34:46.568152+0000","last_unstale":"2024-09-13T14:36:58.369225+0000","last_undegraded":"2024-09-13T14:36:58.369225+0000","last_fullsized":"2024-09-13T14:36:58.369225+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:24:44.592857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":432,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545720+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:37:11.545720+0000","last_peered":"2024-09-13T14:37:11.545720+0000","last_clean":"2024-09-13T14:37:11.545720+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:37:11.545720+0000","last_undegraded":"2024-09-13T14:37:11.545720+0000","last_fullsized":"2024-09-13T14:37:11.545720+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":401,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556388+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:37:11.556388+0000","last_peered":"2024-09-13T14:37:11.556388+0000","last_clean":"2024-09-13T14:37:11.556388+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:37:11.556388+0000","last_undegraded":"2024-09-13T14:37:11.556388+0000","last_fullsized":"2024-09-13T14:37:11.556388+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":365,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547766+0000","last_change":"2024-09-13T14:34:46.624853+0000","last_active":"2024-09-13T14:37:11.547766+0000","last_peered":"2024-09-13T14:37:11.547766+0000","last_clean":"2024-09-13T14:37:11.547766+0000","last_became_active":"2024-09-13T14:34:46.624553+0000","last_became_peered":"2024-09-13T14:34:46.624553+0000","last_unstale":"2024-09-13T14:37:11.547766+0000","last_undegraded":"2024-09-13T14:37:11.547766+0000","last_fullsized":"2024-09-13T14:37:11.547766+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:22.679066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":414,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.545489+0000","last_change":"2024-09-13T14:34:46.567930+0000","last_active":"2024-09-13T14:37:11.545489+0000","last_peered":"2024-09-13T14:37:11.545489+0000","last_clean":"2024-09-13T14:37:11.545489+0000","last_became_active":"2024-09-13T14:34:46.567500+0000","last_became_peered":"2024-09-13T14:34:46.567500+0000","last_unstale":"2024-09-13T14:37:11.545489+0000","last_undegraded":"2024-09-13T14:37:11.545489+0000","last_fullsized":"2024-09-13T14:37:11.545489+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:47.997991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":267,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369109+0000","last_change":"2024-09-13T14:34:46.581764+0000","last_active":"2024-09-13T14:36:58.369109+0000","last_peered":"2024-09-13T14:36:58.369109+0000","last_clean":"2024-09-13T14:36:58.369109+0000","last_became_active":"2024-09-13T14:34:46.581377+0000","last_became_peered":"2024-09-13T14:34:46.581377+0000","last_unstale":"2024-09-13T14:36:58.369109+0000","last_undegraded":"2024-09-13T14:36:58.369109+0000","last_fullsized":"2024-09-13T14:36:58.369109+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:28.920200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":434,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.547236+0000","last_change":"2024-09-13T14:34:46.582553+0000","last_active":"2024-09-13T14:37:11.547236+0000","last_peered":"2024-09-13T14:37:11.547236+0000","last_clean":"2024-09-13T14:37:11.547236+0000","last_became_active":"2024-09-13T14:34:46.582146+0000","last_became_peered":"2024-09-13T14:34:46.582146+0000","last_unstale":"2024-09-13T14:37:11.547236+0000","last_undegraded":"2024-09-13T14:37:11.547236+0000","last_fullsized":"2024-09-13T14:37:11.547236+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:55:13.365210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":392,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.556310+0000","last_change":"2024-09-13T14:21:39.299942+0000","last_active":"2024-09-13T14:37:11.556310+0000","last_peered":"2024-09-13T14:37:11.556310+0000","last_clean":"2024-09-13T14:37:11.556310+0000","last_became_active":"2024-09-13T14:21:39.299537+0000","last_became_peered":"2024-09-13T14:21:39.299537+0000","last_unstale":"2024-09-13T14:37:11.556310+0000","last_undegraded":"2024-09-13T14:37:11.556310+0000","last_fullsized":"2024-09-13T14:37:11.556310+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:35.030971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":265,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.369427+0000","last_change":"2024-09-13T14:34:46.584115+0000","last_active":"2024-09-13T14:36:58.369427+0000","last_peered":"2024-09-13T14:36:58.369427+0000","last_clean":"2024-09-13T14:36:58.369427+0000","last_became_active":"2024-09-13T14:34:46.583779+0000","last_became_peered":"2024-09-13T14:34:46.583779+0000","last_unstale":"2024-09-13T14:36:58.369427+0000","last_undegraded":"2024-09-13T14:36:58.369427+0000","last_fullsized":"2024-09-13T14:36:58.369427+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:19.009728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":250,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.375905+0000","last_change":"2024-09-13T14:34:46.663283+0000","last_active":"2024-09-13T14:36:58.375905+0000","last_peered":"2024-09-13T14:36:58.375905+0000","last_clean":"2024-09-13T14:36:58.375905+0000","last_became_active":"2024-09-13T14:34:46.662866+0000","last_became_peered":"2024-09-13T14:34:46.662866+0000","last_unstale":"2024-09-13T14:36:58.375905+0000","last_undegraded":"2024-09-13T14:36:58.375905+0000","last_fullsized":"2024-09-13T14:36:58.375905+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:17:19.510340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.368995+0000","last_change":"2024-09-13T14:24:00.652085+0000","last_active":"2024-09-13T14:36:58.368995+0000","last_peered":"2024-09-13T14:36:58.368995+0000","last_clean":"2024-09-13T14:36:58.368995+0000","last_became_active":"2024-09-13T14:24:00.651775+0000","last_became_peered":"2024-09-13T14:24:00.651775+0000","last_unstale":"2024-09-13T14:36:58.368995+0000","last_undegraded":"2024-09-13T14:36:58.368995+0000","last_fullsized":"2024-09-13T14:36:58.368995+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:05:05.556810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":216,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.376051+0000","last_change":"2024-09-13T14:13:41.130869+0000","last_active":"2024-09-13T14:36:58.376051+0000","last_peered":"2024-09-13T14:36:58.376051+0000","last_clean":"2024-09-13T14:36:58.376051+0000","last_became_active":"2024-09-13T14:13:41.130589+0000","last_became_peered":"2024-09-13T14:13:41.130589+0000","last_unstale":"2024-09-13T14:36:58.376051+0000","last_undegraded":"2024-09-13T14:36:58.376051+0000","last_fullsized":"2024-09-13T14:36:58.376051+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:41.200123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":424,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.550285+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:37:11.550285+0000","last_peered":"2024-09-13T14:37:11.550285+0000","last_clean":"2024-09-13T14:37:11.550285+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:37:11.550285+0000","last_undegraded":"2024-09-13T14:37:11.550285+0000","last_fullsized":"2024-09-13T14:37:11.550285+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":410,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.546157+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:37:11.546157+0000","last_peered":"2024-09-13T14:37:11.546157+0000","last_clean":"2024-09-13T14:37:11.546157+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:37:11.546157+0000","last_undegraded":"2024-09-13T14:37:11.546157+0000","last_fullsized":"2024-09-13T14:37:11.546157+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":241,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-09-13T14:36:58.368609+0000","last_change":"2024-09-13T14:23:27.720538+0000","last_active":"2024-09-13T14:36:58.368609+0000","last_peered":"2024-09-13T14:36:58.368609+0000","last_clean":"2024-09-13T14:36:58.368609+0000","last_became_active":"2024-09-13T14:23:27.719904+0000","last_became_peered":"2024-09-13T14:23:27.719904+0000","last_unstale":"2024-09-13T14:36:58.368609+0000","last_undegraded":"2024-09-13T14:36:58.368609+0000","last_fullsized":"2024-09-13T14:36:58.368609+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:58.627013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":404,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-09-13T14:37:11.559841+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:37:11.559841+0000","last_peered":"2024-09-13T14:37:11.559841+0000","last_clean":"2024-09-13T14:37:11.559841+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:37:11.559841+0000","last_undegraded":"2024-09-13T14:37:11.559841+0000","last_fullsized":"2024-09-13T14:37:11.559841+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5844893720,"num_objects":89187,"num_object_clones":0,"num_object_copies":267561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89187,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149077,"num_write_kb":7624385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17568641024,"data_stored":17568628808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128277,"ondisk_log_size":128277,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397814,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081196,"kb_used_data":1775524,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91663956,"statfs":{"total":95995035648,"available":93863890944,"internally_reserved":0,"allocated":1818136576,"data_stored":1816812862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594042,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2083296,"kb_used_data":1777624,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91661856,"statfs":{"total":95995035648,"available":93861740544,"internally_reserved":0,"allocated":1820286976,"data_stored":1818974880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790272,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1727640,"kb_used_data":1417808,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92017512,"statfs":{"total":95995035648,"available":94225932288,"internally_reserved":0,"allocated":1451835392,"data_stored":1450527771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986500,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088552,"kb_used_data":1782880,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91656600,"statfs":{"total":95995035648,"available":93856358400,"internally_reserved":0,"allocated":1825669120,"data_stored":1824349432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697057,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3860828,"kb_used_data":3295184,"kb_used_omap":4,"kb_used_meta":565563,"kb_avail":89884324,"statfs":{"total":95995035648,"available":92041547776,"internally_reserved":0,"allocated":3374268416,"data_stored":3372960868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":579136587},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860584,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2612572,"kb_used_data":2175764,"kb_used_omap":8,"kb_used_meta":436599,"kb_avail":91132580,"statfs":{"total":95995035648,"available":93319761920,"internally_reserved":0,"allocated":2227982336,"data_stored":2226671214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":447078220},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020255,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2319128,"kb_used_data":2207312,"kb_used_omap":9,"kb_used_meta":111734,"kb_avail":91426024,"statfs":{"total":95995035648,"available":93620248576,"internally_reserved":0,"allocated":2260287488,"data_stored":2258979349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":114416453},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187646,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3304728,"kb_used_data":2848336,"kb_used_omap":7,"kb_used_meta":456376,"kb_avail":90440424,"statfs":{"total":95995035648,"available":92610994176,"internally_reserved":0,"allocated":2916696064,"data_stored":2915388191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":467329355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2898661376,"data_stored":2898657304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2243104768,"data_stored":2243100696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2219900928,"data_stored":2219900928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3340304384,"data_stored":3340304384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1813905408,"data_stored":1813905408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1445462016,"data_stored":1445462016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1804865536,"data_stored":1804861464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1802436608,"data_stored":1802436608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:37:23.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:37:23.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:23.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:23.540 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:23.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:23.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:23.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:23.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:23.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:23.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:23.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:23.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:24.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:24.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:24.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:24.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:24.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:24.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:24.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:24.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:24.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:24.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:24.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:24.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:24.790 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:24 smithi017 ceph-mon[107556]: pgmap v764: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:24.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:24 smithi017 ceph-mon[103001]: pgmap v764: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:24.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:24.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:24.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:24.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:24.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:24.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:24 smithi189 ceph-mon[90992]: pgmap v764: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:37:25.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:25.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:25.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:25.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:25.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:25.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:25.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:25.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:25.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:25.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:25.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:25.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:25.866 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:25.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:25.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:26.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:26.028 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:26.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:26.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:26.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:26.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:26.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:26.407 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:26.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:26.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:26.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:26.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:26.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:26.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:26.934 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:26.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[103001]: pgmap v765: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-09-13T14:37:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[107556]: pgmap v765: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-09-13T14:37:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:26 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:26 smithi189 ceph-mon[90992]: pgmap v765: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2024-09-13T14:37:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:26 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:27.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:27.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:27.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:27.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:27.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:27.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:27.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:27.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:27.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:27.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:27.603 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:27] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.51.0" 2024-09-13T14:37:27.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:27.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:27.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:27.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:27.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:27.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:27.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:27.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:27.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:28.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:28.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:28.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:28.272 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:28.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:28.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:28.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:28.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:28.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:28.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:28.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:28.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:28.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:28 smithi017 ceph-mon[103001]: pgmap v766: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T14:37:28.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:28 smithi017 ceph-mon[107556]: pgmap v766: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T14:37:28.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:28.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:28.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:28.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:28.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:28.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:28 smithi189 ceph-mon[90992]: pgmap v766: 99 pgs: 99 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T14:37:29.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:29.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:29.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:29.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:29.293 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:29.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:29.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:29.505 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:29.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:29.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:29.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:29.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:29.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:29.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:29.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:29.886 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:29 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:29.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:29.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:29.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:29.941 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:37:29.941 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:37:29.977 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:37:29.977 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:37:30.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:29 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:30.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:29 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:29 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:30.020 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:37:30.020 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:37:30.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:30.062 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:37:30.062 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:37:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:30.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:30.104 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:37:30.104 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:37:30.138 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:37:30.138 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:37:30.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:30.194 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:37:30.194 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:37:30.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:30.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:37:30.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:37:30.290 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:37:30.290 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:37:30.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:30.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:30.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:30.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:30.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:30.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:30.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:30.768 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:30 smithi189 ceph-mon[90992]: pgmap v767: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:37:30.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/634400894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:30.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:30.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:30.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:30.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:30.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:30 smithi017 ceph-mon[103001]: pgmap v767: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:37:30.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/634400894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:30.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:30 smithi017 ceph-mon[107556]: pgmap v767: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:37:30.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/634400894' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:31.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:31.049 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:31.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:31.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:31.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:31.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:31.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:31.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:31.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:37:31.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:31.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:31.628 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:37:31.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4119099398' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:31 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4119099398' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:31 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:31.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:31.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:31.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:31.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:31.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4119099398' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:31 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:32.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:32.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:32.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:32.286 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:32.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:32.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:32.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:32.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:32.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:32.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:32.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:32.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:32.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:32.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:32.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:32.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:32.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[103001]: pgmap v768: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[103001]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[107556]: pgmap v768: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:37:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:32 smithi017 ceph-mon[107556]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:37:32.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:32 smithi189 ceph-mon[90992]: pgmap v768: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:37:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:32 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:37:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:32 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:37:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:32 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:37:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:32 smithi189 ceph-mon[90992]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:37:33.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:33.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:33.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:33.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:33.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:33.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:33.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:33.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:33.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:33.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:33.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:33.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:33.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:33 smithi017 ceph-mon[103001]: pgmap v770: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-09-13T14:37:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:33 smithi017 ceph-mon[107556]: pgmap v770: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-09-13T14:37:33.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:33.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:34.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:33 smithi189 ceph-mon[90992]: pgmap v770: 99 pgs: 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-09-13T14:37:34.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:37:34.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:34.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:34.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:34.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:34.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:34.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:34.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:34.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:34.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:34.623 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:34.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:34.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:34.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:34.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:34.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:34.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:35.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:35.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:35.154 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:35.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:35.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:35.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:35.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:35.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:35.539 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:35.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:35.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:35.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:35.809 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:35.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:35.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:35.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:36.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:36.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:36.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:37:36.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:36.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:36.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:36.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:36.330 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:36.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:36 smithi017 ceph-mon[103001]: pgmap v771: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T14:37:36.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:36 smithi017 ceph-mon[107556]: pgmap v771: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T14:37:36.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:36.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:36.516 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:36 smithi189 ceph-mon[90992]: pgmap v771: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T14:37:36.616 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:36.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:36.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:36.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:36.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:36.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:36.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:37.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:37.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:37.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:37.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:37.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:37.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:37.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:37.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:37.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:37.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:37.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:37.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:37.661 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:37] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.51.0" 2024-09-13T14:37:37.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:37.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:37.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:37.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:37.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:37.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:38.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:38.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:38.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:38.134 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-13T14:37:38.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:38.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:38.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:38.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:38.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:38.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:38 smithi017 ceph-mon[107556]: pgmap v772: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:37:38.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:37:38.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:38 smithi017 ceph-mon[103001]: pgmap v772: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:37:38.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:37:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:38 smithi189 ceph-mon[90992]: pgmap v772: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:37:38.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:37:38.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:38.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:38.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:38.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:38.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:38.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:38.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:38.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:38.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:39.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:39.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:39.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:39.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:39.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:37:39.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:39 smithi017 ceph-mon[107556]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:37:39.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:37:39.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:39 smithi017 ceph-mon[103001]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:37:39.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4121121471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:37:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:39 smithi189 ceph-mon[90992]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:37:39.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:39.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:39.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:39.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:39.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:39.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:39.853 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:39.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:39.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:39.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:39.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:40.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:40.116 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:37:40.117 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:37:40.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:40.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:40.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:40.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:40.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:40 smithi017 ceph-mon[103001]: pgmap v774: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:37:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:40 smithi017 ceph-mon[107556]: pgmap v774: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:37:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:40 smithi189 ceph-mon[90992]: pgmap v774: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-13T14:37:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:40.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:40.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:40.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:40.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:40.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:40.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:40.924 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:40.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:40.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:41.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:41.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:41.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:41.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:41.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:41.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:41.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:41.481 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:41.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:41.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:41.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:41.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:41.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:41.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:41.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:41.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:41.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:42.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:42.096 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:37:42.097 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:37:42.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:42.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:37:42.135 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:37:42.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:37:42.172 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:37:42.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:37:42.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:37:42.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:42.243 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:37:42.243 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:37:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:42.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:42.278 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:37:42.278 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:37:42.308 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:37:42.308 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:37:42.340 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:37:42.340 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:37:42.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:42.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:42.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[103001]: pgmap v775: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:37:42.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/514072519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:42.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:42.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[107556]: pgmap v775: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:37:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/514072519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:42.461 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:42 smithi189 ceph-mon[90992]: pgmap v775: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:37:42.461 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/514072519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:42.461 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:42.461 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:42.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:42.530 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:42.631 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:42.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:42.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:42.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:42.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:42.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:42.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:43.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:43.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:43.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:43.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:43.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:43.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:43.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:43.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:43.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:43.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:43.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:43.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:43.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:37:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:43.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:43.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:44.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:44.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:44.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:44.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:44.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:44.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:44 smithi017 ceph-mon[103001]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-09-13T14:37:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:44 smithi017 ceph-mon[107556]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-09-13T14:37:44.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:44.473 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:37:44.473 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:37:44.474 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-13T14:37:44.474 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:37:44.474 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:37:44.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-13T14:37:44.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:44.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:44 smithi189 ceph-mon[90992]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-09-13T14:37:44.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:44.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:44.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:44.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:44.830 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:37:44.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:44.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:44.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:44.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:45.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:45.047 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:45.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:45.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:45.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:45.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:45.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:45.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:45.571 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:45.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:45.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:45.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:45.737 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:45.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:45.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:45.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:45.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:46.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:46.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:46.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:46.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:46.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:46.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:46.385 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:46 smithi189 ceph-mon[90992]: pgmap v777: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:37:46.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:46.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:46 smithi017 ceph-mon[103001]: pgmap v777: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:37:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:46 smithi017 ceph-mon[107556]: pgmap v777: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:37:46.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:46.578 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:46.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:46.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:46.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:37:46.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:46.737 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:46.755 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:37:46.756 INFO:teuthology.orchestra.run.smithi189.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:37:46.756 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:37:46.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:46.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:46.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:46.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:47.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:47.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:47.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:47.195 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:37:47.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:47.278 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:47.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:47.384 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:37:47.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:37:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:47] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:37:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:37:47.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:47.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:47.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:47.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:47.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:47.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:47.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:47.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:48.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:48.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:48.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:48.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:48.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:48.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:48.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:48.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[103001]: pgmap v778: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-13T14:37:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:37:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[103001]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:37:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[107556]: pgmap v778: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-13T14:37:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:37:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:48 smithi017 ceph-mon[107556]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:37:48.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:48 smithi189 ceph-mon[90992]: pgmap v778: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-13T14:37:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:48 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:37:48.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3598190062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:37:48.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:48 smithi189 ceph-mon[90992]: osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:37:48.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:48.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:48.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:48.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:48.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:48.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:48.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:48.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:49.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:49.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:49.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:49.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:49.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:49.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:49.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:49.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:49.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:49.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:49.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:49.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:49.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:49.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:49.822 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:37:49.822 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:37:49.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240679 5.18 5:18e01e66:::benchmark_data_smithi017_147622_object106050:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.476943+0000", 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.339458536, 2024-09-13T14:37:49.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.166644676, 2024-09-13T14:37:49.824 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.824 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.824 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.824 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.825 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.825 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.825 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240679 2024-09-13T14:37:49.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.825 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476943+0000", 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476943+0000", 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476944+0000", 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4400000000000001e-07 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476945+0000", 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8220000000000001e-06 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476951+0000", 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8239999999999996e-06 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476956+0000", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1070000000000004e-06 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478294+0000", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013378400000000001 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478301+0000", 2024-09-13T14:37:49.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5300000000000002e-06 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.629703+0000", 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.151401973 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.629838+0000", 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013547200000000001 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.630317+0000", 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047875499999999998 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.633641+0000", 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033245029999999999 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643054+0000", 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0094128379999999998 2024-09-13T14:37:49.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643096+0000", 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1721000000000003e-05 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643224+0000", 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012810299999999999 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643300+0000", 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5606999999999996e-05 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643390+0000", 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0031000000000006e-05 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643588+0000", 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019820600000000001 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.832 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240705 5.18 5:1dc0362e:::benchmark_data_smithi017_147622_object106076:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.476962+0000", 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.339439513, 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1745678989999999, 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.833 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240705 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476962+0000", 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476962+0000", 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476963+0000", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4299999999999999e-07 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476964+0000", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.694e-06 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476970+0000", 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9050000000000002e-06 2024-09-13T14:37:49.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476975+0000", 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.921e-06 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478323+0000", 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013481820000000001 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478329+0000", 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9630000000000003e-06 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.633671+0000", 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1553413800000001 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.633774+0000", 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103844 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.634386+0000", 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00061119600000000005 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643111+0000", 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087254870000000005 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643169+0000", 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8205999999999998e-05 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643636+0000", 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046619199999999998 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651174+0000", 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075385979999999997 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651279+0000", 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010488399999999999 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651397+0000", 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118387 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651530+0000", 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132717 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.842 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240718 5.18 5:1ea949cc:::benchmark_data_smithi017_147622_object106089:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.476981+0000", 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33942089599998, 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.175325768, 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.843 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240718 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476981+0000", 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476981+0000", 2024-09-13T14:37:49.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476981+0000", 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2599999999999998e-07 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476983+0000", 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7889999999999999e-06 2024-09-13T14:37:49.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476989+0000", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8139999999999997e-06 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.476994+0000", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8409999999999999e-06 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478353+0000", 2024-09-13T14:37:49.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013591110000000001 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478360+0000", 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5750000000000003e-06 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.638026+0000", 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1596660759999999 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.638158+0000", 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131786 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.638651+0000", 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049239399999999995 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643649+0000", 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049983019999999996 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643697+0000", 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7794999999999999e-05 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643736+0000", 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9486000000000001e-05 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651226+0000", 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074901919999999997 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651603+0000", 2024-09-13T14:37:49.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037681200000000002 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651661+0000", 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7485000000000003e-05 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652307+0000", 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064598399999999995 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240719 5.18 5:1ac57828:::benchmark_data_smithi017_147622_object106090:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.852 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477000+0000", 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.339401243, 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.175820946, 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.853 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240719 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477000+0000", 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477000+0000", 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477001+0000", 2024-09-13T14:37:49.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.72e-07 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477003+0000", 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0080000000000001e-06 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477010+0000", 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6140000000000003e-06 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477016+0000", 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9989999999999997e-06 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478383+0000", 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013665509999999999 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478389+0000", 2024-09-13T14:37:49.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643765+0000", 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1653758089999999 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.643888+0000", 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000123213 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.644431+0000", 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054348099999999996 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.651127+0000", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066959130000000004 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652379+0000", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012513629999999999 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652454+0000", 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5129999999999994e-05 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652517+0000", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3754e-05 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652639+0000", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121835 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652701+0000", 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1413000000000003e-05 2024-09-13T14:37:49.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652821+0000", 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012075500000000001 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240725 5.18 5:182b1a6f:::benchmark_data_smithi017_147622_object106096:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477059+0000", 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33934260699999, 2024-09-13T14:37:49.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.187631023, 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240725 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477059+0000", 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477059+0000", 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477060+0000", 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0999999999999999e-07 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477062+0000", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3750000000000001e-06 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477068+0000", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5529999999999998e-06 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477074+0000", 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4909999999999996e-06 2024-09-13T14:37:49.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478411+0000", 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001336612 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478417+0000", 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3709999999999996e-06 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.647607+0000", 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1691896159999999 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.647750+0000", 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014340300000000001 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.648213+0000", 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000462595 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652535+0000", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043219900000000004 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652593+0000", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8360999999999999e-05 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652895+0000", 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030195999999999999 2024-09-13T14:37:49.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664435+0000", 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011540079 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664483+0000", 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7892999999999997e-05 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664553+0000", 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9933999999999998e-05 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664690+0000", 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013737999999999999 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240781 5.18 5:1fc9a581:::benchmark_data_smithi017_147622_object106152:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.872 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477082+0000", 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.339319832, 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.20841931, 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240781 2024-09-13T14:37:49.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477082+0000", 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477082+0000", 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477082+0000", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477084+0000", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2199999999999999e-06 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477091+0000", 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7499999999999997e-06 2024-09-13T14:37:49.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477098+0000", 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0010000000000004e-06 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478438+0000", 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001340106 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478444+0000", 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3700000000000003e-06 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.652951+0000", 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.174507186 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.653604+0000", 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00065310699999999999 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.654101+0000", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049671600000000002 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664382+0000", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010281568 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664741+0000", 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035907700000000002 2024-09-13T14:37:49.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.672906+0000", 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081642480000000007 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.672991+0000", 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4920999999999996e-05 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685067+0000", 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012076191 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685274+0000", 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000207519 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685501+0000", 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000226999 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.880 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240799 5.18 5:1c559ae0:::benchmark_data_smithi017_147622_object106170:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477105+0000", 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33929706800001, 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.208704832, 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.881 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240799 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477105+0000", 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477105+0000", 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477105+0000", 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7399999999999999e-07 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477112+0000", 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5139999999999998e-06 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477118+0000", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.976e-06 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477123+0000", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2669999999999997e-06 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478464+0000", 2024-09-13T14:37:49.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013415149999999999 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478471+0000", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3820000000000001e-06 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.656937+0000", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1784661590000001 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.657059+0000", 2024-09-13T14:37:49.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122626 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.657483+0000", 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042323300000000002 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664389+0000", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069067060000000003 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.672798+0000", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084090659999999998 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.672948+0000", 2024-09-13T14:37:49.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014939400000000001 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673143+0000", 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019478799999999999 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685637+0000", 2024-09-13T14:37:49.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012494168 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685694+0000", 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7055999999999997e-05 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685810+0000", 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011560799999999999 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.889 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240844 5.18 5:1f235e58:::benchmark_data_smithi017_147622_object106215:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477131+0000", 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33927093, 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.208898284, 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240844 2024-09-13T14:37:49.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477131+0000", 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477131+0000", 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477131+0000", 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.32e-07 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477134+0000", 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4789999999999999e-06 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477140+0000", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6800000000000004e-06 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477146+0000", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.276e-06 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478493+0000", 2024-09-13T14:37:49.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001347153 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478499+0000", 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1580000000000002e-06 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.660684+0000", 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.182185225 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.660853+0000", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000169023 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.661310+0000", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045677199999999998 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.672846+0000", 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011535971000000001 2024-09-13T14:37:49.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673084+0000", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000237736 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673276+0000", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019200399999999999 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673365+0000", 2024-09-13T14:37:49.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9462999999999994e-05 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685849+0000", 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012484006000000001 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685912+0000", 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2810000000000003e-05 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686029+0000", 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011709600000000001 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240860 5.18 5:1f73853a:::benchmark_data_smithi017_147622_object106231:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477152+0000", 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33924967299998, 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2090922710000001, 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.898 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240860 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477152+0000", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477152+0000", 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477152+0000", 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6399999999999998e-07 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477154+0000", 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8780000000000001e-06 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477160+0000", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8189999999999997e-06 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477165+0000", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8620000000000002e-06 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478520+0000", 2024-09-13T14:37:49.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013550979999999999 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478526+0000", 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1279999999999996e-06 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.664787+0000", 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1862602980000001 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.665066+0000", 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027989200000000001 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.665591+0000", 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052469799999999998 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673067+0000", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074758689999999996 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673102+0000", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4551000000000002e-05 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673325+0000", 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022319500000000001 2024-09-13T14:37:49.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673414+0000", 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.92e-05 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686071+0000", 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012657180000000001 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686123+0000", 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1969999999999999e-05 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686244+0000", 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012126899999999999 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.910 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240932 5.18 5:1b37dfa6:::benchmark_data_smithi017_147622_object106303:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477170+0000", 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33923126299999, 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2092910720000001, 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.911 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240932 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477170+0000", 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477170+0000", 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477171+0000", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3799999999999998e-07 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477173+0000", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7400000000000001e-06 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477179+0000", 2024-09-13T14:37:49.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5529999999999998e-06 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477184+0000", 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3040000000000004e-06 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478547+0000", 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013627470000000001 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478553+0000", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0689999999999998e-06 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.669112+0000", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.190558961 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.669260+0000", 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000148214 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.669782+0000", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052170199999999997 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673193+0000", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034110680000000002 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673458+0000", 2024-09-13T14:37:49.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026509 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673503+0000", 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5018000000000002e-05 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.684994+0000", 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011490314 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686290+0000", 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001296148 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686342+0000", 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2067000000000001e-05 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686462+0000", 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119739 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.918 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35034.0:240982 5.18 5:1b94acd9:::benchmark_data_smithi017_147622_object106353:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:29:19.477190+0000", 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 510.33921162000001, 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2094714689999999, 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35034", 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2947465279", 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 240982 2024-09-13T14:37:49.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477190+0000", 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477190+0000", 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477190+0000", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2899999999999999e-07 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477192+0000", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.812e-06 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477198+0000", 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7350000000000001e-06 2024-09-13T14:37:49.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.477204+0000", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.071e-06 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478574+0000", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013703910000000001 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:19.478581+0000", 2024-09-13T14:37:49.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1380000000000004e-06 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673539+0000", 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1949579260000001 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.673706+0000", 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016718899999999999 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4,6", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.674212+0000", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050627500000000004 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685005+0000", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010792728999999999 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685034+0000", 2024-09-13T14:37:49.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.942e-05 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685554+0000", 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051980400000000001 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.685596+0000", 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2018000000000003e-05 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686503+0000", 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090697300000000005 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686552+0000", 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8916999999999999e-05 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:29:20.686662+0000", 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010974199999999999 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.927 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214850 6.19 6:9efaf998:::benchmark_data_smithi017_185083_object90238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32919966700001, 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1610996929999999, 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.928 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214850 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6e-07 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487204+0000", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2890000000000001e-06 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487207+0000", 2024-09-13T14:37:49.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7160000000000001e-06 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487210+0000", 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0410000000000002e-06 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534594+0000", 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047383458000000003 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534603+0000", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4509999999999998e-06 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632356+0000", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0977522900000001 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632494+0000", 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138255 2024-09-13T14:37:49.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632917+0000", 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042332999999999997 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640108+0000", 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071911279999999998 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640419+0000", 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031022999999999999 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640470+0000", 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1239999999999997e-05 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640785+0000", 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031557999999999998 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640827+0000", 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1998e-05 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648079+0000", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007251955 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648150+0000", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1000999999999993e-05 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648302+0000", 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151471 2024-09-13T14:37:49.936 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214865 6.19 6:990a3470:::benchmark_data_smithi017_185083_object90253:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32918613000001, 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.161541361, 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214865 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.03e-07 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.940 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:49.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487217+0000", 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1200000000000001e-06 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487221+0000", 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9670000000000003e-06 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487224+0000", 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7829999999999999e-06 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534628+0000", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.04740403 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534634+0000", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636252+0000", 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.101618046 2024-09-13T14:37:49.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636387+0000", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013459400000000001 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636762+0000", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037534800000000001 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640114+0000", 2024-09-13T14:37:49.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033514040000000001 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640868+0000", 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075420200000000004 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640935+0000", 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7180000000000004e-05 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640981+0000", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5516000000000003e-05 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641023+0000", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2682000000000002e-05 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648552+0000", 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075286629999999997 2024-09-13T14:37:49.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648628+0000", 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6207999999999999e-05 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648757+0000", 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128854 2024-09-13T14:37:49.946 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214872 6.19 6:98033914:::benchmark_data_smithi017_185083_object90260:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.329139943, 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170118998, 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.947 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214872 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487264+0000", 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479999999999998e-06 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487269+0000", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2870000000000004e-06 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487274+0000", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8269999999999997e-06 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534661+0000", 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047386510999999999 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534667+0000", 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0959999999999997e-06 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641055+0000", 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1063884900000001 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641187+0000", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013205500000000001 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641618+0000", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000430772 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648355+0000", 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006736952 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648368+0000", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2619000000000001e-05 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648401+0000", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3334000000000003e-05 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648438+0000", 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7083999999999999e-05 2024-09-13T14:37:49.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648476+0000", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7651000000000003e-05 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657179+0000", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087027679999999996 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657239+0000", 2024-09-13T14:37:49.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0705000000000002e-05 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657381+0000", 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014156800000000001 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214886 6.19 6:9efbdfd4:::benchmark_data_smithi017_185083_object90274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32911159099999, 2024-09-13T14:37:49.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170313833, 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214886 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:37:49.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1e-07 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487292+0000", 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7269999999999999e-06 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487296+0000", 2024-09-13T14:37:49.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7069999999999998e-06 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487299+0000", 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9490000000000001e-06 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534693+0000", 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047393909999999997 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534699+0000", 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0889999999999996e-06 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.644543+0000", 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1098440000000001 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.644779+0000", 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023615299999999999 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.645439+0000", 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00065987700000000001 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648512+0000", 2024-09-13T14:37:49.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030732929999999999 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656850+0000", 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083375059999999997 2024-09-13T14:37:49.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656918+0000", 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8691999999999999e-05 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656972+0000", 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3526999999999998e-05 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657014+0000", 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2472000000000003e-05 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657426+0000", 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00041171300000000002 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657478+0000", 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2320000000000001e-05 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657604+0000", 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012558799999999999 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214932 6.19 6:9ca5d594:::benchmark_data_smithi017_185083_object90320:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32909863200001, 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170562039, 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.968 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214932 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2100000000000001e-07 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487308+0000", 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.634e-06 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487313+0000", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9479999999999999e-06 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487316+0000", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8100000000000002e-06 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534724+0000", 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047408690000000003 2024-09-13T14:37:49.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534732+0000", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7989999999999994e-06 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648787+0000", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1140544859999999 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648928+0000", 2024-09-13T14:37:49.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141581 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.649304+0000", 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037605400000000001 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656933+0000", 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076289690000000002 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657056+0000", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012271999999999999 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657096+0000", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9823000000000001e-05 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657138+0000", 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2122000000000001e-05 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657648+0000", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051048300000000005 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657690+0000", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2032e-05 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657740+0000", 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9747999999999999e-05 2024-09-13T14:37:49.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657865+0000", 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124919 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214964 6.19 6:9cdc11f4:::benchmark_data_smithi017_185083_object90352:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32906684299999, 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1837455560000001, 2024-09-13T14:37:49.976 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214964 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6899999999999999e-07 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487337+0000", 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3960000000000001e-06 2024-09-13T14:37:49.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487341+0000", 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.464e-06 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487344+0000", 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7750000000000001e-06 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534759+0000", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047414748999999999 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534765+0000", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6479999999999997e-06 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.651636+0000", 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1168708469999999 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.651763+0000", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126546 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.654562+0000", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027995910000000001 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660645+0000", 2024-09-13T14:37:49.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060826470000000001 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660659+0000", 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4129e-05 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667519+0000", 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068605159999999997 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667636+0000", 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116562 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667672+0000", 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5710000000000002e-05 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670895+0000", 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032235300000000001 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670948+0000", 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2732999999999998e-05 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671080+0000", 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013244400000000001 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.984 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214989 6.19 6:9b173040:::benchmark_data_smithi017_185083_object90377:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32905369299999, 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1840168639999999, 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214989 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487350+0000", 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.72e-06 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487355+0000", 2024-09-13T14:37:49.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3539999999999999e-06 2024-09-13T14:37:49.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487361+0000", 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1379999999999998e-06 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534793+0000", 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047432329000000002 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534799+0000", 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1720000000000004e-06 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657906+0000", 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.123107012 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.658034+0000", 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127694 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.658392+0000", 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035811099999999999 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.666020+0000", 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076280389999999997 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667562+0000", 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015422929999999999 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667707+0000", 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014456199999999999 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667742+0000", 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4925999999999997e-05 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671119+0000", 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033768280000000001 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671190+0000", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1470999999999994e-05 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671245+0000", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4509000000000003e-05 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671365+0000", 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120375 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:49.994 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:215071 6.19 6:9b4254d5:::benchmark_data_smithi017_185083_object90459:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.32903439, 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.184973252, 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 215071 2024-09-13T14:37:49.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:49.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487368+0000", 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4799999999999999e-07 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487370+0000", 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479999999999998e-06 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487375+0000", 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7939999999999999e-06 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487380+0000", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8470000000000003e-06 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534825+0000", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047444403000000003 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534831+0000", 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2090000000000002e-06 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660686+0000", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1258555370000001 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660810+0000", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012359900000000001 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.661169+0000", 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035927600000000001 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:49.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667600+0000", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006430247 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667776+0000", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000176603 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671155+0000", 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003378944 2024-09-13T14:37:50.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671400+0000", 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024505499999999999 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672123+0000", 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072296399999999999 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672161+0000", 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7564999999999997e-05 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672217+0000", 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5998999999999997e-05 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672341+0000", 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012391400000000001 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:50.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:215109 6.19 6:9a809a80:::benchmark_data_smithi017_185083_object90497:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.329013815, 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1883943699999999, 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 215109 2024-09-13T14:37:50.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:37:50.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.84e-07 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487390+0000", 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1069999999999999e-06 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:37:50.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487397+0000", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7499999999999997e-06 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487402+0000", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0819999999999998e-06 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534856+0000", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047453433000000003 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:37:50.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534862+0000", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663372+0000", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.128510023 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663490+0000", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118367 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:37:50.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663825+0000", 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000334111 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670848+0000", 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070239309999999998 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672351+0000", 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015024109999999999 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672421+0000", 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0005000000000005e-05 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672461+0000", 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0558e-05 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675488+0000", 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030265019999999999 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675579+0000", 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1113999999999995e-05 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675643+0000", 2024-09-13T14:37:50.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3744999999999994e-05 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675782+0000", 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001395 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:37:50.011 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:37:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:50.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:50.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:50.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:50.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:50.268 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:50.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:50.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:50 smithi017 ceph-mon[103001]: pgmap v780: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:50 smithi017 ceph-mon[107556]: pgmap v780: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:50 smithi189 ceph-mon[90992]: pgmap v780: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T14:37:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:37:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:37:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:50.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:50.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:50.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:50.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:50.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:50.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:50.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:50.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:50.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:51.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:51.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:51.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:51.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:37:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:51.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:51.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:51.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:51.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:51.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:51.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:37:51.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:51.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:51.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:51.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:51.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:51.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:51.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:52.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:52.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:52.200 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:52.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:52.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:52.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:52.424 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:37:52.424 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:37:52.424 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:37:52.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd out 4 2024-09-13T14:37:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:52 smithi017 ceph-mon[103001]: pgmap v781: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-09-13T14:37:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:52 smithi017 ceph-mon[107556]: pgmap v781: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-09-13T14:37:52.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:52.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:52 smithi189 ceph-mon[90992]: pgmap v781: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-09-13T14:37:52.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:52.624 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:52.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:52.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:52.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:52.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:52.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:53.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:53.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:53.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:53.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:53.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:53.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:53.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:53.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:53.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:53.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:53.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:53.699 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:53.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:53.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:53.920 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:53.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:53.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:53.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:53.965 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:54.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:37:54.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:54.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:37:54.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:54.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:54.337 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:54.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:54 smithi017 ceph-mon[103001]: pgmap v782: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-13T14:37:54.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:54 smithi017 ceph-mon[107556]: pgmap v782: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-13T14:37:54.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:54.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:54 smithi189 ceph-mon[90992]: pgmap v782: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-09-13T14:37:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:54.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:54.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:54.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:54.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:54.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:54.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:54.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:54.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:55.022 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:37:55.022 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:37:55.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:37:55.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:37:55.026 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:37:55.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:55.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:55.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:55.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:55.245 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:55.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:55.354 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:55.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4090656367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:55.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:55.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4090656367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:55.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4090656367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:37:55.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:55.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:55.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:55.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:55.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:55.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:55.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:55.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:56.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:56.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:56.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:56.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:56.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:56.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:56.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:56 smithi017 ceph-mon[103001]: pgmap v783: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:56 smithi017 ceph-mon[107556]: pgmap v783: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:56.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:56.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:56 smithi189 ceph-mon[90992]: pgmap v783: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:37:56.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:56.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:56.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:56.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:56.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:56.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:56.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:56.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:56.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:57.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:57.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:57.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:57.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:57.356 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:57.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:57.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:37:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:57.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:37:57.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:57.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:37:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:37:57] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.51.0" 2024-09-13T14:37:57.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:57.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:57.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:57.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:37:57.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:57.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:37:58.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:58.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:58.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:58.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:58.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:58.304 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[103001]: pgmap v784: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:37:58.304 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:58.304 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:58.304 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[107556]: pgmap v784: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:37:58.305 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:58.305 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:58.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:58.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:58.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:58 smithi189 ceph-mon[90992]: pgmap v784: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:37:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:37:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:58.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:58.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:58.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:58.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:58.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:58.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:58.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:58.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:59.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:37:59.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:37:59.121 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:37:59.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:59.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:37:59.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:59.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:59 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1452074350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:37:59 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:59 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1452074350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:37:59 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:37:59.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:37:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:59 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1452074350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:37:59 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:37:59.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:37:59.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:59.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:37:59.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:37:59.757 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:37:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:37:59.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:37:59.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:37:59.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:37:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:37:59.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:00.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:00.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:00.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:00.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:00.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:00.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:00.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:00.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:00 smithi189 ceph-mon[90992]: pgmap v785: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-13T14:38:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:00 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:00 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:00 smithi189 ceph-mon[90992]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[103001]: pgmap v785: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[103001]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[107556]: pgmap v785: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:00.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:00 smithi017 ceph-mon[107556]: osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:38:00.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:00.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:00.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:00.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:00.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:00.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:00.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:00.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:00.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:00.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:01.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:01.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:01.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:01.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:01.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:01.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:01.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:01.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:01.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:01.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:01.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:01.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:01.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:01.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:01.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:01.944 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:38:01.944 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:38:02.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:02.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:02.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:02.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:02 smithi017 ceph-mon[107556]: pgmap v787: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-09-13T14:38:02.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:02 smithi017 ceph-mon[103001]: pgmap v787: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-09-13T14:38:02.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:02.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:02.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:02.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:02 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T14:38:02.195+0000 7f0b3f9c1640 -1 mon.a@0(leader).osd e271 definitely_dead 0 2024-09-13T14:38:02.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:02 smithi189 ceph-mon[90992]: pgmap v787: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2024-09-13T14:38:02.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:02.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:38:02.631 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:02.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:02.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:02.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:02.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:03.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:03.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:03.142 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.4. 2024-09-13T14:38:03.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:03.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:03.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:03.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:03.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:38:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:03 smithi017 ceph-mon[103001]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:38:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:38:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:03 smithi017 ceph-mon[107556]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:38:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:03.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:38:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:03 smithi189 ceph-mon[90992]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:38:03.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:38:03.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:03.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:03.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:03.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:03.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:03.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:03.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:03.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:03.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:04.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:04.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:04.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:04.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:04.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:04.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:04.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[103001]: pgmap v788: 99 pgs: 99 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[103001]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[103001]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[107556]: pgmap v788: 99 pgs: 99 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[107556]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:38:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:04 smithi017 ceph-mon[107556]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:38:04.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:04 smithi189 ceph-mon[90992]: pgmap v788: 99 pgs: 99 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:38:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2086035799' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:38:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:04 smithi189 ceph-mon[90992]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:38:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:04 smithi189 ceph-mon[90992]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:38:04.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:04.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:04.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:04.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:04.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:04.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:04.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:04.922 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:05.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:05.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:05.173 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:05.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:05.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:05.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:05.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:05.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:05.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:05.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:05.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 271'11562 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 271'11627 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11317 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 7.8 starting backfill to osd.3 from (0'0,0'0] MAX to 271'11758 2024-09-13T14:38:05.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[103001]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11664 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 271'11562 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 271'11627 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11317 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 7.8 starting backfill to osd.3 from (0'0,0'0] MAX to 271'11758 2024-09-13T14:38:05.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:05 smithi017 ceph-mon[107556]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11664 2024-09-13T14:38:05.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:05.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:05.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:05.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:05.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:05.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:38:05.989 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:38:06.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:38:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:38:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 271'11562 2024-09-13T14:38:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 271'11627 2024-09-13T14:38:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11317 2024-09-13T14:38:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 7.8 starting backfill to osd.3 from (0'0,0'0] MAX to 271'11758 2024-09-13T14:38:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:05 smithi189 ceph-mon[90992]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 271'11664 2024-09-13T14:38:06.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:38:06.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:38:06.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:06.084 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:06.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:38:06.086 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:38:06.140 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:38:06.141 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:38:06.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:06.178 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:38:06.178 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:38:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:06.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:06.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:38:06.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:38:06.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:38:06.241 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:38:06.280 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:38:06.280 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:38:06.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:06.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:06.448 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:06.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:06.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:06.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:06.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:06.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:06.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:06.882 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:06.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[107556]: pgmap v792: 99 pgs: 7 peering, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:38:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:38:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[107556]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:38:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3031424243' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[103001]: pgmap v792: 99 pgs: 7 peering, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:38:06.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:38:06.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[103001]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:38:06.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3031424243' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:06.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:06 smithi189 ceph-mon[90992]: pgmap v792: 99 pgs: 7 peering, 92 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T14:38:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:06 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:38:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:06 smithi189 ceph-mon[90992]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:38:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3031424243' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:07.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:07.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:07.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:07.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:07.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:07.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:07.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:07.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:07.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:07.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:07] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.51.0" 2024-09-13T14:38:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:07 smithi017 ceph-mon[103001]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:38:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:07 smithi017 ceph-mon[107556]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:38:07.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:07.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:07.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:07.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:07.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:07.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:07 smithi189 ceph-mon[90992]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:38:08.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:08.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:08.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:08.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:08.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:08.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:08.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:08.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:08.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:08.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:08.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:08.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:08 smithi017 ceph-mon[107556]: pgmap v795: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 7 peering, 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 17 KiB/s rd, 38 KiB/s wr, 581 op/s; 9726/157473 objects misplaced (6.176%); 2.0 MiB/s, 33 objects/s recovering 2024-09-13T14:38:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:08 smithi017 ceph-mon[103001]: pgmap v795: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 7 peering, 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 17 KiB/s rd, 38 KiB/s wr, 581 op/s; 9726/157473 objects misplaced (6.176%); 2.0 MiB/s, 33 objects/s recovering 2024-09-13T14:38:08.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:08.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:08.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:08.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:08.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:08 smithi189 ceph-mon[90992]: pgmap v795: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 7 peering, 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 17 KiB/s rd, 38 KiB/s wr, 581 op/s; 9726/157473 objects misplaced (6.176%); 2.0 MiB/s, 33 objects/s recovering 2024-09-13T14:38:09.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:09.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:09.144 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:38:09.144 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:38:09.145 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2024-09-13T14:38:09.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:38:09.203 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:09.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:09.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:09.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:09.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:09.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:09.610 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:38:09.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:09.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:09.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:09.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:09.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:09.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:09.928 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:09.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1886298058' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1886298058' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:09.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:38:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1886298058' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:38:10.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:10.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:10.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:10.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:10.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:10.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:10.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:10.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:10.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:10.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:10.704 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:10.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:10.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:10.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:10.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:10.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: pgmap v796: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 475 op/s; 15489/152889 objects misplaced (10.131%); 6.8 MiB/s, 1 keys/s, 114 objects/s recovering 2024-09-13T14:38:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:38:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[103001]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: pgmap v796: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 475 op/s; 15489/152889 objects misplaced (10.131%); 6.8 MiB/s, 1 keys/s, 114 objects/s recovering 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:38:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:10 smithi017 ceph-mon[107556]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:38:10.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:11.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: pgmap v796: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 475 op/s; 15489/152889 objects misplaced (10.131%); 6.8 MiB/s, 1 keys/s, 114 objects/s recovering 2024-09-13T14:38:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:38:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:38:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:38:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:10 smithi189 ceph-mon[90992]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:38:11.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:11.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:11.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:11.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:11.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:11.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:11.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:11.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:11.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:11.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:11.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:11.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:11.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:11.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:11.926 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:11.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:12.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:12.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:12.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:12.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:12.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:12.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:12.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:12.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:12.473 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:12.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:12.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:12.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:12.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:12.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:12.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:12.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:12 smithi017 ceph-mon[103001]: pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 612 op/s; 14637/149199 objects misplaced (9.810%); 10 MiB/s, 1 keys/s, 166 objects/s recovering 2024-09-13T14:38:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:12 smithi017 ceph-mon[107556]: pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 612 op/s; 14637/149199 objects misplaced (9.810%); 10 MiB/s, 1 keys/s, 166 objects/s recovering 2024-09-13T14:38:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:12.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:12.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:12.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:12 smithi189 ceph-mon[90992]: pgmap v798: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 612 op/s; 14637/149199 objects misplaced (9.810%); 10 MiB/s, 1 keys/s, 166 objects/s recovering 2024-09-13T14:38:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:13.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:13.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:13.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:13.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:13.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:13.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:13.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:13.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:13.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:13.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:13.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:13.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:13.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:13.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:13.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:13.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:14.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:14.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:14.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:14.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:14.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:14.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:14.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:14.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:14.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:14.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:14.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:14 smithi017 ceph-mon[103001]: pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.1 KiB/s rd, 20 KiB/s wr, 560 op/s; 14000/145851 objects misplaced (9.599%); 12 MiB/s, 1 keys/s, 187 objects/s recovering 2024-09-13T14:38:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:14 smithi017 ceph-mon[107556]: pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.1 KiB/s rd, 20 KiB/s wr, 560 op/s; 14000/145851 objects misplaced (9.599%); 12 MiB/s, 1 keys/s, 187 objects/s recovering 2024-09-13T14:38:14.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:14.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:14.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:14.959 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:14.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:14.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:14 smithi189 ceph-mon[90992]: pgmap v799: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 9.1 KiB/s rd, 20 KiB/s wr, 560 op/s; 14000/145851 objects misplaced (9.599%); 12 MiB/s, 1 keys/s, 187 objects/s recovering 2024-09-13T14:38:15.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:15.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:15.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:15.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:15.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:15.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:15.397 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:15.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:15.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:15.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:15.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:15.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:15.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:15 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:38:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:15 smithi017 ceph-mon[103001]: 1.5 scrub ok 2024-09-13T14:38:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:15 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:38:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:15 smithi017 ceph-mon[107556]: 1.5 scrub ok 2024-09-13T14:38:15.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:15.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:15.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:15.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:15.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:15.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:15 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:38:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:15 smithi189 ceph-mon[90992]: 1.5 scrub ok 2024-09-13T14:38:16.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:16.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:16.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:16.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:16.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:16.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:16.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:16.426 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:38:16.427 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:38:16.427 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:38:16.427 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:38:16.427 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:38:16.448 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:16.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:16.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:16.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:16.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:16.685 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:16 smithi017 ceph-mon[103001]: pgmap v800: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 8.0 KiB/s rd, 17 KiB/s wr, 661 op/s; 12836/141609 objects misplaced (9.064%); 16 MiB/s, 1 keys/s, 256 objects/s recovering 2024-09-13T14:38:16.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:16 smithi017 ceph-mon[107556]: pgmap v800: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 8.0 KiB/s rd, 17 KiB/s wr, 661 op/s; 12836/141609 objects misplaced (9.064%); 16 MiB/s, 1 keys/s, 256 objects/s recovering 2024-09-13T14:38:16.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:16.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:16.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:16.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:16.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:16.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:16 smithi189 ceph-mon[90992]: pgmap v800: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 8.0 KiB/s rd, 17 KiB/s wr, 661 op/s; 12836/141609 objects misplaced (9.064%); 16 MiB/s, 1 keys/s, 256 objects/s recovering 2024-09-13T14:38:17.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:17.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:17.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:17.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:17.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:38:17.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:17.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:17.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:17.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:17.561 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:17] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-09-13T14:38:17.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:17.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:17.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:17.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:17.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:17.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:38:17.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:17.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:17.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:18.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:18.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:18.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:18.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:18.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:18.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:18.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:18.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:18.467 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:18.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:18.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:18.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:18 smithi017 ceph-mon[103001]: pgmap v801: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 11585/137715 objects misplaced (8.412%); 17 MiB/s, 0 keys/s, 277 objects/s recovering 2024-09-13T14:38:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:18 smithi017 ceph-mon[107556]: pgmap v801: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 11585/137715 objects misplaced (8.412%); 17 MiB/s, 0 keys/s, 277 objects/s recovering 2024-09-13T14:38:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:18.709 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:18.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:18 smithi189 ceph-mon[90992]: pgmap v801: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 11585/137715 objects misplaced (8.412%); 17 MiB/s, 0 keys/s, 277 objects/s recovering 2024-09-13T14:38:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:18.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:18.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:18.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:18.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:19.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:19.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:19.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:19.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:19.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:19.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:19.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:19.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:19.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:19.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:19.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:19.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:19.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:19.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:19.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:19.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:19.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:20.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:20.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:20.200 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:20.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:20.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:20.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:20.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:20.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:20 smithi189 ceph-mon[90992]: pgmap v802: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 10467/131742 objects misplaced (7.945%); 19 MiB/s, 306 objects/s recovering 2024-09-13T14:38:20.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:20.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:20 smithi017 ceph-mon[103001]: pgmap v802: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 10467/131742 objects misplaced (7.945%); 19 MiB/s, 306 objects/s recovering 2024-09-13T14:38:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:20 smithi017 ceph-mon[107556]: pgmap v802: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 10467/131742 objects misplaced (7.945%); 19 MiB/s, 306 objects/s recovering 2024-09-13T14:38:20.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:20.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:20.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:20.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:20.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:20.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:21.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:21.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:21.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:21.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:21.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:21.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:21.440 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:21.453 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:21.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:21.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:21.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:21.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:21.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:21.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:21.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:21.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:22.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:22.040 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:22.142 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:22.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:22.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:22.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:22.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:22.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:22.441 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:22.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:22.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:22.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:22 smithi017 ceph-mon[103001]: pgmap v803: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 9650/128043 objects misplaced (7.537%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:38:22.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:22 smithi017 ceph-mon[107556]: pgmap v803: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 9650/128043 objects misplaced (7.537%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:38:22.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:22.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:22 smithi189 ceph-mon[90992]: pgmap v803: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 720 op/s; 9650/128043 objects misplaced (7.537%); 19 MiB/s, 303 objects/s recovering 2024-09-13T14:38:22.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:22.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:22.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:22.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:23.006 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:23.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:23.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:23.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:23.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:23.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:23.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:23.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:23.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:23.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:23.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:23.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:23.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:23.903 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:23.933 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:38:23.935 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:38:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:23.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:23.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:24.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:24.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:24.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:24.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:24.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:24.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:24.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:24.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:24.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:24.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:24.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:24 smithi017 ceph-mon[107556]: pgmap v804: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 9236/124554 objects misplaced (7.415%); 18 MiB/s, 281 objects/s recovering 2024-09-13T14:38:24.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:24 smithi017 ceph-mon[107556]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:38:24.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:24 smithi017 ceph-mon[103001]: pgmap v804: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 9236/124554 objects misplaced (7.415%); 18 MiB/s, 281 objects/s recovering 2024-09-13T14:38:24.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:24 smithi017 ceph-mon[103001]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:38:24.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:24 smithi189 ceph-mon[90992]: pgmap v804: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.5 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 9236/124554 objects misplaced (7.415%); 18 MiB/s, 281 objects/s recovering 2024-09-13T14:38:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:24 smithi189 ceph-mon[90992]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:38:24.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:24.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:24.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:24.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:24.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:25.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:25.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:25.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:25.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:25.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:25.303 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:25.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:25.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:25.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:25.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:25.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:25.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:25.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:25.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:25.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:25.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:25.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:26.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:26.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:26.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:26.214 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:26.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:26.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:26.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:26.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:26.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:26.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:26.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:26 smithi017 ceph-mon[103001]: pgmap v805: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 8274/119490 objects misplaced (6.924%); 19 MiB/s, 301 objects/s recovering 2024-09-13T14:38:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:26 smithi017 ceph-mon[103001]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:38:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:26 smithi017 ceph-mon[107556]: pgmap v805: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 8274/119490 objects misplaced (6.924%); 19 MiB/s, 301 objects/s recovering 2024-09-13T14:38:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:26 smithi017 ceph-mon[107556]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:38:26.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:26.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:26 smithi189 ceph-mon[90992]: pgmap v805: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 8274/119490 objects misplaced (6.924%); 19 MiB/s, 301 objects/s recovering 2024-09-13T14:38:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:26 smithi189 ceph-mon[90992]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:38:26.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:26.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:26.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:26.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:26.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:27.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:27.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:27.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:27.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:27.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:27.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:27.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:27.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:27.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:27.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:27] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-13T14:38:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:27 smithi017 ceph-mon[103001]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:38:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:27 smithi017 ceph-mon[107556]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:38:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:27 smithi189 ceph-mon[90992]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:38:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:27.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:27.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:27.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:27.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:27.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:27.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:28.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:28.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:28.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:28.212 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:28.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:28.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:28.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:28.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:28.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:28.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:28 smithi017 ceph-mon[103001]: pgmap v808: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 6901/115467 objects misplaced (5.977%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:38:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:28 smithi017 ceph-mon[107556]: pgmap v808: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 6901/115467 objects misplaced (5.977%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:38:28.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:28.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:28 smithi189 ceph-mon[90992]: pgmap v808: 99 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 94 active+clean; 2.4 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 6901/115467 objects misplaced (5.977%); 18 MiB/s, 282 objects/s recovering 2024-09-13T14:38:28.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:28.967 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:28.993 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:28.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:29.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:29.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:29.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:29.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:29.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:29.402 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:29.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:29.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:29.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:29.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:29.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:29.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:29.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:29.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:29.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:29.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:30.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:30.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:30.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:30.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:30.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:30.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:30.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:30.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:30.421 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:38:30.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:38:30.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:30.439 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:38:30.439 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:38:30.455 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:38:30.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:38:30.457 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:30.497 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:38:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:38:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:30 smithi189 ceph-mon[90992]: pgmap v809: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 5711/110088 objects misplaced (5.188%); 22 MiB/s, 358 objects/s recovering 2024-09-13T14:38:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:30 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1189476482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:30.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:38:30.523 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:38:30.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:38:30.543 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:38:30.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:30.563 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:38:30.563 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:38:30.583 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:38:30.583 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:38:30.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[103001]: pgmap v809: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 5711/110088 objects misplaced (5.188%); 22 MiB/s, 358 objects/s recovering 2024-09-13T14:38:30.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:30.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1189476482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:30.584 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[107556]: pgmap v809: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 5711/110088 objects misplaced (5.188%); 22 MiB/s, 358 objects/s recovering 2024-09-13T14:38:30.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:30.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1189476482' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:30.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:30.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:30.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:30.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:30.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:30.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:30.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:30.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:31.077 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:31.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:31.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:31.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:31.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:31.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:31.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:31.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:31.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:31.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:31.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:31.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:31.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:31.942 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:31.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:31.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:32.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:32.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:32.202 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":804,"stamp":"2024-09-13T14:38:23.084600+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2723813349,"num_objects":41518,"num_object_clones":0,"num_object_copies":124554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9236,"num_objects_unfound":0,"num_objects_dirty":41518,"num_whiteouts":0,"num_read":5999,"num_read_kb":13012,"num_write":197681,"num_write_kb":7637008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4382,"num_bytes_recovered":285020954,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113866,"ondisk_log_size":113866,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10167284,"kb_used_data":7503484,"kb_used_omap":34,"kb_used_meta":2663581,"kb_avail":646048780,"statfs":{"total":671965249536,"available":661553950720,"internally_reserved":0,"allocated":7683567616,"data_stored":7673967114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":2727506991},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,2,11,5,11,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-538431910,"num_objects":-8215,"num_object_clones":0,"num_object_copies":-24645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-5401,"num_objects_unfound":0,"num_objects_dirty":-8215,"num_whiteouts":0,"num_read":-27,"num_read_kb":-68,"num_write":8201,"num_write_kb":-148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3374,"num_bytes_recovered":221064794,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003320"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":435,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634602+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:38:09.634602+0000","last_peered":"2024-09-13T14:38:09.634602+0000","last_clean":"2024-09-13T14:38:09.634602+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:38:09.634602+0000","last_undegraded":"2024-09-13T14:38:09.634602+0000","last_fullsized":"2024-09-13T14:38:09.634602+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":259,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577013+0000","last_change":"2024-09-13T14:38:03.566972+0000","last_active":"2024-09-13T14:38:06.577013+0000","last_peered":"2024-09-13T14:38:06.577013+0000","last_clean":"2024-09-13T14:38:06.577013+0000","last_became_active":"2024-09-13T14:38:03.561212+0000","last_became_peered":"2024-09-13T14:38:03.561212+0000","last_unstale":"2024-09-13T14:38:06.577013+0000","last_undegraded":"2024-09-13T14:38:06.577013+0000","last_fullsized":"2024-09-13T14:38:06.577013+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:41.501833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":240,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592621+0000","last_change":"2024-09-13T14:24:00.653974+0000","last_active":"2024-09-13T14:38:06.592621+0000","last_peered":"2024-09-13T14:38:06.592621+0000","last_clean":"2024-09-13T14:38:06.592621+0000","last_became_active":"2024-09-13T14:24:00.653483+0000","last_became_peered":"2024-09-13T14:24:00.653483+0000","last_unstale":"2024-09-13T14:38:06.592621+0000","last_undegraded":"2024-09-13T14:38:06.592621+0000","last_fullsized":"2024-09-13T14:38:06.592621+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:49.003360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":420,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630647+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:38:09.630647+0000","last_peered":"2024-09-13T14:38:09.630647+0000","last_clean":"2024-09-13T14:38:09.630647+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:38:09.630647+0000","last_undegraded":"2024-09-13T14:38:09.630647+0000","last_fullsized":"2024-09-13T14:38:09.630647+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":420,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630714+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:38:09.630714+0000","last_peered":"2024-09-13T14:38:09.630714+0000","last_clean":"2024-09-13T14:38:09.630714+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:38:09.630714+0000","last_undegraded":"2024-09-13T14:38:09.630714+0000","last_fullsized":"2024-09-13T14:38:09.630714+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":234,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577177+0000","last_change":"2024-09-13T14:38:03.566493+0000","last_active":"2024-09-13T14:38:06.577177+0000","last_peered":"2024-09-13T14:38:06.577177+0000","last_clean":"2024-09-13T14:38:06.577177+0000","last_became_active":"2024-09-13T14:38:03.566174+0000","last_became_peered":"2024-09-13T14:38:03.566174+0000","last_unstale":"2024-09-13T14:38:06.577177+0000","last_undegraded":"2024-09-13T14:38:06.577177+0000","last_fullsized":"2024-09-13T14:38:06.577177+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.777867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":434,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.638677+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:38:09.638677+0000","last_peered":"2024-09-13T14:38:09.638677+0000","last_clean":"2024-09-13T14:38:09.638677+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:38:09.638677+0000","last_undegraded":"2024-09-13T14:38:09.638677+0000","last_fullsized":"2024-09-13T14:38:09.638677+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":249,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582115+0000","last_change":"2024-09-13T14:21:39.300587+0000","last_active":"2024-09-13T14:38:06.582115+0000","last_peered":"2024-09-13T14:38:06.582115+0000","last_clean":"2024-09-13T14:38:06.582115+0000","last_became_active":"2024-09-13T14:21:39.300131+0000","last_became_peered":"2024-09-13T14:21:39.300131+0000","last_unstale":"2024-09-13T14:38:06.582115+0000","last_undegraded":"2024-09-13T14:38:06.582115+0000","last_fullsized":"2024-09-13T14:38:06.582115+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:48.292958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"277'12284","reported_seq":12356,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:19.776146+0000","last_change":"2024-09-13T14:34:57.524042+0000","last_active":"2024-09-13T14:38:19.776146+0000","last_peered":"2024-09-13T14:38:19.776146+0000","last_clean":"2024-09-13T14:38:19.776146+0000","last_became_active":"2024-09-13T14:34:57.523748+0000","last_became_peered":"2024-09-13T14:34:57.523748+0000","last_unstale":"2024-09-13T14:38:19.776146+0000","last_undegraded":"2024-09-13T14:38:19.776146+0000","last_fullsized":"2024-09-13T14:38:19.776146+0000","mapping_epoch":247,"log_start":"259'5966","ondisk_log_start":"259'5966","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:42.905684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171835392,"num_objects":2622,"num_object_clones":0,"num_object_copies":7866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12284,"num_write_kb":476992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":256,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577457+0000","last_change":"2024-09-13T14:38:03.617493+0000","last_active":"2024-09-13T14:38:06.577457+0000","last_peered":"2024-09-13T14:38:06.577457+0000","last_clean":"2024-09-13T14:38:06.577457+0000","last_became_active":"2024-09-13T14:38:03.569854+0000","last_became_peered":"2024-09-13T14:38:03.569854+0000","last_unstale":"2024-09-13T14:38:06.577457+0000","last_undegraded":"2024-09-13T14:38:06.577457+0000","last_fullsized":"2024-09-13T14:38:06.577457+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:17.890012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":223,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592297+0000","last_change":"2024-09-13T14:13:42.489030+0000","last_active":"2024-09-13T14:38:06.592297+0000","last_peered":"2024-09-13T14:38:06.592297+0000","last_clean":"2024-09-13T14:38:06.592297+0000","last_became_active":"2024-09-13T14:13:42.488693+0000","last_became_peered":"2024-09-13T14:13:42.488693+0000","last_unstale":"2024-09-13T14:38:06.592297+0000","last_undegraded":"2024-09-13T14:38:06.592297+0000","last_fullsized":"2024-09-13T14:38:06.592297+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.929831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"277'12517","reported_seq":12589,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:20.226146+0000","last_change":"2024-09-13T14:34:57.522300+0000","last_active":"2024-09-13T14:38:20.226146+0000","last_peered":"2024-09-13T14:38:20.226146+0000","last_clean":"2024-09-13T14:38:20.226146+0000","last_became_active":"2024-09-13T14:34:57.522005+0000","last_became_peered":"2024-09-13T14:34:57.522005+0000","last_unstale":"2024-09-13T14:38:20.226146+0000","last_undegraded":"2024-09-13T14:38:20.226146+0000","last_fullsized":"2024-09-13T14:38:20.226146+0000","mapping_epoch":247,"log_start":"261'6565","ondisk_log_start":"261'6565","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:56.018499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166658048,"num_objects":2543,"num_object_clones":0,"num_object_copies":7629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12517,"num_write_kb":481920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":274,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577555+0000","last_change":"2024-09-13T14:38:03.622235+0000","last_active":"2024-09-13T14:38:06.577555+0000","last_peered":"2024-09-13T14:38:06.577555+0000","last_clean":"2024-09-13T14:38:06.577555+0000","last_became_active":"2024-09-13T14:38:03.570116+0000","last_became_peered":"2024-09-13T14:38:03.570116+0000","last_unstale":"2024-09-13T14:38:06.577555+0000","last_undegraded":"2024-09-13T14:38:06.577555+0000","last_fullsized":"2024-09-13T14:38:06.577555+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:24.128120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.644979+0000","last_change":"2024-09-13T14:38:03.571764+0000","last_active":"2024-09-13T14:38:09.644979+0000","last_peered":"2024-09-13T14:38:09.644979+0000","last_clean":"2024-09-13T14:38:09.644979+0000","last_became_active":"2024-09-13T14:38:03.571230+0000","last_became_peered":"2024-09-13T14:38:03.571230+0000","last_unstale":"2024-09-13T14:38:09.644979+0000","last_undegraded":"2024-09-13T14:38:09.644979+0000","last_fullsized":"2024-09-13T14:38:09.644979+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:17.921751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"277'12209","reported_seq":16360,"reported_epoch":277,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:38:18.468397+0000","last_change":"2024-09-13T14:38:04.873979+0000","last_active":"2024-09-13T14:38:18.468397+0000","last_peered":"2024-09-13T14:38:18.468397+0000","last_clean":"2024-09-13T14:38:03.084304+0000","last_became_active":"2024-09-13T14:38:04.708899+0000","last_became_peered":"2024-09-13T14:38:04.708899+0000","last_unstale":"2024-09-13T14:38:18.468397+0000","last_undegraded":"2024-09-13T14:38:18.468397+0000","last_fullsized":"2024-09-13T14:38:18.468397+0000","mapping_epoch":273,"log_start":"258'5344","ondisk_log_start":"258'5344","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6865,"log_dups_size":0,"ondisk_log_size":6865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":841,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12209,"num_write_kb":476480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2681}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":225,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577263+0000","last_change":"2024-09-13T14:13:41.131762+0000","last_active":"2024-09-13T14:38:06.577263+0000","last_peered":"2024-09-13T14:38:06.577263+0000","last_clean":"2024-09-13T14:38:06.577263+0000","last_became_active":"2024-09-13T14:13:41.131405+0000","last_became_peered":"2024-09-13T14:13:41.131405+0000","last_unstale":"2024-09-13T14:38:06.577263+0000","last_undegraded":"2024-09-13T14:38:06.577263+0000","last_fullsized":"2024-09-13T14:38:06.577263+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:36.343926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":257,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582252+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:38:06.582252+0000","last_peered":"2024-09-13T14:38:06.582252+0000","last_clean":"2024-09-13T14:38:06.582252+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:38:06.582252+0000","last_undegraded":"2024-09-13T14:38:06.582252+0000","last_fullsized":"2024-09-13T14:38:06.582252+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"277'12563","reported_seq":14732,"reported_epoch":277,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:38:21.873761+0000","last_change":"2024-09-13T14:38:04.813721+0000","last_active":"2024-09-13T14:38:21.873761+0000","last_peered":"2024-09-13T14:38:21.873761+0000","last_clean":"2024-09-13T14:38:03.091220+0000","last_became_active":"2024-09-13T14:38:04.727516+0000","last_became_peered":"2024-09-13T14:38:04.727516+0000","last_unstale":"2024-09-13T14:38:21.873761+0000","last_undegraded":"2024-09-13T14:38:21.873761+0000","last_fullsized":"2024-09-13T14:38:21.873761+0000","mapping_epoch":273,"log_start":"255'3735","ondisk_log_start":"255'3735","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1695,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12563,"num_write_kb":484800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1046,"num_bytes_recovered":68550656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":2587}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":241,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592649+0000","last_change":"2024-09-13T14:24:00.654168+0000","last_active":"2024-09-13T14:38:06.592649+0000","last_peered":"2024-09-13T14:38:06.592649+0000","last_clean":"2024-09-13T14:38:06.592649+0000","last_became_active":"2024-09-13T14:24:00.653921+0000","last_became_peered":"2024-09-13T14:24:00.653921+0000","last_unstale":"2024-09-13T14:38:06.592649+0000","last_undegraded":"2024-09-13T14:38:06.592649+0000","last_fullsized":"2024-09-13T14:38:06.592649+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:26.165634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":439,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631200+0000","last_change":"2024-09-13T14:38:03.576970+0000","last_active":"2024-09-13T14:38:09.631200+0000","last_peered":"2024-09-13T14:38:09.631200+0000","last_clean":"2024-09-13T14:38:09.631200+0000","last_became_active":"2024-09-13T14:38:03.576495+0000","last_became_peered":"2024-09-13T14:38:03.576495+0000","last_unstale":"2024-09-13T14:38:09.631200+0000","last_undegraded":"2024-09-13T14:38:09.631200+0000","last_fullsized":"2024-09-13T14:38:09.631200+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:52.911056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"277'12169","reported_seq":12240,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:18.465737+0000","last_change":"2024-09-13T14:34:57.526751+0000","last_active":"2024-09-13T14:38:18.465737+0000","last_peered":"2024-09-13T14:38:18.465737+0000","last_clean":"2024-09-13T14:38:18.465737+0000","last_became_active":"2024-09-13T14:34:57.526442+0000","last_became_peered":"2024-09-13T14:34:57.526442+0000","last_unstale":"2024-09-13T14:38:18.465737+0000","last_undegraded":"2024-09-13T14:38:18.465737+0000","last_fullsized":"2024-09-13T14:38:18.465737+0000","mapping_epoch":247,"log_start":"257'5266","ondisk_log_start":"257'5266","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:09.948770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172032000,"num_objects":2625,"num_object_clones":0,"num_object_copies":7875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12169,"num_write_kb":473408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":388,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.576915+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:38:06.576915+0000","last_peered":"2024-09-13T14:38:06.576915+0000","last_clean":"2024-09-13T14:38:06.576915+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:38:06.576915+0000","last_undegraded":"2024-09-13T14:38:06.576915+0000","last_fullsized":"2024-09-13T14:38:06.576915+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":433,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.636466+0000","last_change":"2024-09-13T14:38:03.614233+0000","last_active":"2024-09-13T14:38:09.636466+0000","last_peered":"2024-09-13T14:38:09.636466+0000","last_clean":"2024-09-13T14:38:09.636466+0000","last_became_active":"2024-09-13T14:38:03.578168+0000","last_became_peered":"2024-09-13T14:38:03.578168+0000","last_unstale":"2024-09-13T14:38:09.636466+0000","last_undegraded":"2024-09-13T14:38:09.636466+0000","last_fullsized":"2024-09-13T14:38:09.636466+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:59.664708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634993+0000","last_change":"2024-09-13T14:38:03.568127+0000","last_active":"2024-09-13T14:38:09.634993+0000","last_peered":"2024-09-13T14:38:09.634993+0000","last_clean":"2024-09-13T14:38:09.634993+0000","last_became_active":"2024-09-13T14:38:03.567585+0000","last_became_peered":"2024-09-13T14:38:03.567585+0000","last_unstale":"2024-09-13T14:38:09.634993+0000","last_undegraded":"2024-09-13T14:38:09.634993+0000","last_fullsized":"2024-09-13T14:38:09.634993+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:42.094924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"277'12411","reported_seq":12483,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:21.736233+0000","last_change":"2024-09-13T14:34:57.525034+0000","last_active":"2024-09-13T14:38:21.736233+0000","last_peered":"2024-09-13T14:38:21.736233+0000","last_clean":"2024-09-13T14:38:21.736233+0000","last_became_active":"2024-09-13T14:34:57.524740+0000","last_became_peered":"2024-09-13T14:34:57.524740+0000","last_unstale":"2024-09-13T14:38:21.736233+0000","last_undegraded":"2024-09-13T14:38:21.736233+0000","last_fullsized":"2024-09-13T14:38:21.736233+0000","mapping_epoch":247,"log_start":"254'3552","ondisk_log_start":"254'3552","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":8859,"log_dups_size":0,"ondisk_log_size":8859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:24:44.966638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164560896,"num_objects":2511,"num_object_clones":0,"num_object_copies":7533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12411,"num_write_kb":477504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":135,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592218+0000","last_change":"2024-09-13T14:30:42.699976+0000","last_active":"2024-09-13T14:38:06.592218+0000","last_peered":"2024-09-13T14:38:06.592218+0000","last_clean":"2024-09-13T14:38:06.592218+0000","last_became_active":"2024-09-13T14:30:11.460972+0000","last_became_peered":"2024-09-13T14:30:11.460972+0000","last_unstale":"2024-09-13T14:38:06.592218+0000","last_undegraded":"2024-09-13T14:38:06.592218+0000","last_fullsized":"2024-09-13T14:38:06.592218+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:44.017760+0000","scrub_duration":0.002840637,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":268,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582440+0000","last_change":"2024-09-13T14:38:03.571256+0000","last_active":"2024-09-13T14:38:06.582440+0000","last_peered":"2024-09-13T14:38:06.582440+0000","last_clean":"2024-09-13T14:38:06.582440+0000","last_became_active":"2024-09-13T14:38:03.570928+0000","last_became_peered":"2024-09-13T14:38:03.570928+0000","last_unstale":"2024-09-13T14:38:06.582440+0000","last_undegraded":"2024-09-13T14:38:06.582440+0000","last_fullsized":"2024-09-13T14:38:06.582440+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:42.569492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.610029+0000","last_change":"2024-09-13T14:38:03.566161+0000","last_active":"2024-09-13T14:38:09.610029+0000","last_peered":"2024-09-13T14:38:09.610029+0000","last_clean":"2024-09-13T14:38:09.610029+0000","last_became_active":"2024-09-13T14:38:03.565469+0000","last_became_peered":"2024-09-13T14:38:03.565469+0000","last_unstale":"2024-09-13T14:38:09.610029+0000","last_undegraded":"2024-09-13T14:38:09.610029+0000","last_fullsized":"2024-09-13T14:38:09.610029+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:02.172363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"277'12498","reported_seq":12536,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:23.000621+0000","last_change":"2024-09-13T14:34:57.522071+0000","last_active":"2024-09-13T14:38:23.000621+0000","last_peered":"2024-09-13T14:38:23.000621+0000","last_clean":"2024-09-13T14:38:23.000621+0000","last_became_active":"2024-09-13T14:34:57.521798+0000","last_became_peered":"2024-09-13T14:34:57.521798+0000","last_unstale":"2024-09-13T14:38:23.000621+0000","last_undegraded":"2024-09-13T14:38:23.000621+0000","last_fullsized":"2024-09-13T14:38:23.000621+0000","mapping_epoch":247,"log_start":"258'5770","ondisk_log_start":"258'5770","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:47.723539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164036632,"num_objects":2504,"num_object_clones":0,"num_object_copies":7512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2504,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12498,"num_write_kb":480001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.608946+0000","last_change":"2024-09-13T14:38:03.574077+0000","last_active":"2024-09-13T14:38:09.608946+0000","last_peered":"2024-09-13T14:38:09.608946+0000","last_clean":"2024-09-13T14:38:09.608946+0000","last_became_active":"2024-09-13T14:38:03.573731+0000","last_became_peered":"2024-09-13T14:38:03.573731+0000","last_unstale":"2024-09-13T14:38:09.608946+0000","last_undegraded":"2024-09-13T14:38:09.608946+0000","last_fullsized":"2024-09-13T14:38:09.608946+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.341352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582609+0000","last_change":"2024-09-13T14:38:03.573525+0000","last_active":"2024-09-13T14:38:06.582609+0000","last_peered":"2024-09-13T14:38:06.582609+0000","last_clean":"2024-09-13T14:38:06.582609+0000","last_became_active":"2024-09-13T14:38:03.572863+0000","last_became_peered":"2024-09-13T14:38:03.572863+0000","last_unstale":"2024-09-13T14:38:06.582609+0000","last_undegraded":"2024-09-13T14:38:06.582609+0000","last_fullsized":"2024-09-13T14:38:06.582609+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:04:10.340538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"277'12025","reported_seq":12063,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:22.980478+0000","last_change":"2024-09-13T14:34:57.521324+0000","last_active":"2024-09-13T14:38:22.980478+0000","last_peered":"2024-09-13T14:38:22.980478+0000","last_clean":"2024-09-13T14:38:22.980478+0000","last_became_active":"2024-09-13T14:34:57.520992+0000","last_became_peered":"2024-09-13T14:34:57.520992+0000","last_unstale":"2024-09-13T14:38:22.980478+0000","last_undegraded":"2024-09-13T14:38:22.980478+0000","last_fullsized":"2024-09-13T14:38:22.980478+0000","mapping_epoch":247,"log_start":"258'5336","ondisk_log_start":"258'5336","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.105872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157745152,"num_objects":2407,"num_object_clones":0,"num_object_copies":7221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12025,"num_write_kb":461824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":128,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592297+0000","last_change":"2024-09-13T14:30:39.679683+0000","last_active":"2024-09-13T14:38:06.592297+0000","last_peered":"2024-09-13T14:38:06.592297+0000","last_clean":"2024-09-13T14:38:06.592297+0000","last_became_active":"2024-09-13T14:30:11.460347+0000","last_became_peered":"2024-09-13T14:30:11.460347+0000","last_unstale":"2024-09-13T14:38:06.592297+0000","last_undegraded":"2024-09-13T14:38:06.592297+0000","last_fullsized":"2024-09-13T14:38:06.592297+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:40.415282+0000","scrub_duration":0.002548023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":436,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.632119+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:38:09.632119+0000","last_peered":"2024-09-13T14:38:09.632119+0000","last_clean":"2024-09-13T14:38:09.632119+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:38:09.632119+0000","last_undegraded":"2024-09-13T14:38:09.632119+0000","last_fullsized":"2024-09-13T14:38:09.632119+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":391,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.635127+0000","last_change":"2024-09-13T14:38:03.563815+0000","last_active":"2024-09-13T14:38:09.635127+0000","last_peered":"2024-09-13T14:38:09.635127+0000","last_clean":"2024-09-13T14:38:09.635127+0000","last_became_active":"2024-09-13T14:38:03.563464+0000","last_became_peered":"2024-09-13T14:38:03.563464+0000","last_unstale":"2024-09-13T14:38:09.635127+0000","last_undegraded":"2024-09-13T14:38:09.635127+0000","last_fullsized":"2024-09-13T14:38:09.635127+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:01.276859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"277'12042","reported_seq":12114,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:18.466270+0000","last_change":"2024-09-13T14:34:57.525058+0000","last_active":"2024-09-13T14:38:18.466270+0000","last_peered":"2024-09-13T14:38:18.466270+0000","last_clean":"2024-09-13T14:38:18.466270+0000","last_became_active":"2024-09-13T14:34:57.524514+0000","last_became_peered":"2024-09-13T14:34:57.524514+0000","last_unstale":"2024-09-13T14:38:18.466270+0000","last_undegraded":"2024-09-13T14:38:18.466270+0000","last_fullsized":"2024-09-13T14:38:18.466270+0000","mapping_epoch":247,"log_start":"257'5163","ondisk_log_start":"257'5163","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:17.836817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180092928,"num_objects":2748,"num_object_clones":0,"num_object_copies":8244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12042,"num_write_kb":473280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":379,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577369+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:38:06.577369+0000","last_peered":"2024-09-13T14:38:06.577369+0000","last_clean":"2024-09-13T14:38:06.577369+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:38:06.577369+0000","last_undegraded":"2024-09-13T14:38:06.577369+0000","last_fullsized":"2024-09-13T14:38:06.577369+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":842,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.594005+0000","last_change":"2024-09-13T14:23:27.720548+0000","last_active":"2024-09-13T14:38:06.594005+0000","last_peered":"2024-09-13T14:38:06.594005+0000","last_clean":"2024-09-13T14:38:06.594005+0000","last_became_active":"2024-09-13T14:23:27.720064+0000","last_became_peered":"2024-09-13T14:23:27.720064+0000","last_unstale":"2024-09-13T14:38:06.594005+0000","last_undegraded":"2024-09-13T14:38:06.594005+0000","last_fullsized":"2024-09-13T14:38:06.594005+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:59.412134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":858,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":378,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.637137+0000","last_change":"2024-09-13T14:24:00.653446+0000","last_active":"2024-09-13T14:38:09.637137+0000","last_peered":"2024-09-13T14:38:09.637137+0000","last_clean":"2024-09-13T14:38:09.637137+0000","last_became_active":"2024-09-13T14:24:00.653001+0000","last_became_peered":"2024-09-13T14:24:00.653001+0000","last_unstale":"2024-09-13T14:38:09.637137+0000","last_undegraded":"2024-09-13T14:38:09.637137+0000","last_fullsized":"2024-09-13T14:38:09.637137+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:06.139756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":233,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582837+0000","last_change":"2024-09-13T14:38:03.568729+0000","last_active":"2024-09-13T14:38:06.582837+0000","last_peered":"2024-09-13T14:38:06.582837+0000","last_clean":"2024-09-13T14:38:06.582837+0000","last_became_active":"2024-09-13T14:38:03.568266+0000","last_became_peered":"2024-09-13T14:38:03.568266+0000","last_unstale":"2024-09-13T14:38:06.582837+0000","last_undegraded":"2024-09-13T14:38:06.582837+0000","last_fullsized":"2024-09-13T14:38:06.582837+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:50.779935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"277'12295","reported_seq":12333,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:18.088960+0000","last_change":"2024-09-13T14:34:57.525197+0000","last_active":"2024-09-13T14:38:18.088960+0000","last_peered":"2024-09-13T14:38:18.088960+0000","last_clean":"2024-09-13T14:38:18.088960+0000","last_became_active":"2024-09-13T14:34:57.524916+0000","last_became_peered":"2024-09-13T14:34:57.524916+0000","last_unstale":"2024-09-13T14:38:18.088960+0000","last_undegraded":"2024-09-13T14:38:18.088960+0000","last_fullsized":"2024-09-13T14:38:18.088960+0000","mapping_epoch":247,"log_start":"257'5074","ondisk_log_start":"257'5074","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":7221,"log_dups_size":0,"ondisk_log_size":7221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:01.282676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174653440,"num_objects":2665,"num_object_clones":0,"num_object_copies":7995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12295,"num_write_kb":478720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":231,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.637221+0000","last_change":"2024-09-13T14:38:03.575175+0000","last_active":"2024-09-13T14:38:09.637221+0000","last_peered":"2024-09-13T14:38:09.637221+0000","last_clean":"2024-09-13T14:38:09.637221+0000","last_became_active":"2024-09-13T14:38:03.574520+0000","last_became_peered":"2024-09-13T14:38:03.574520+0000","last_unstale":"2024-09-13T14:38:09.637221+0000","last_undegraded":"2024-09-13T14:38:09.637221+0000","last_fullsized":"2024-09-13T14:38:09.637221+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:32.812111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":667,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577316+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:38:06.577316+0000","last_peered":"2024-09-13T14:38:06.577316+0000","last_clean":"2024-09-13T14:38:06.577316+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:38:06.577316+0000","last_undegraded":"2024-09-13T14:38:06.577316+0000","last_fullsized":"2024-09-13T14:38:06.577316+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":428,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.645835+0000","last_change":"2024-09-13T14:38:03.572417+0000","last_active":"2024-09-13T14:38:09.645835+0000","last_peered":"2024-09-13T14:38:09.645835+0000","last_clean":"2024-09-13T14:38:09.645835+0000","last_became_active":"2024-09-13T14:38:03.571496+0000","last_became_peered":"2024-09-13T14:38:03.571496+0000","last_unstale":"2024-09-13T14:38:09.645835+0000","last_undegraded":"2024-09-13T14:38:09.645835+0000","last_fullsized":"2024-09-13T14:38:09.645835+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:29.444056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":443,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630975+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:38:09.630975+0000","last_peered":"2024-09-13T14:38:09.630975+0000","last_clean":"2024-09-13T14:38:09.630975+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:38:09.630975+0000","last_undegraded":"2024-09-13T14:38:09.630975+0000","last_fullsized":"2024-09-13T14:38:09.630975+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"277'12270","reported_seq":12348,"reported_epoch":277,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:38:18.315620+0000","last_change":"2024-09-13T14:38:04.859279+0000","last_active":"2024-09-13T14:38:18.315620+0000","last_peered":"2024-09-13T14:38:18.315620+0000","last_clean":"2024-09-13T14:38:03.117187+0000","last_became_active":"2024-09-13T14:38:04.705748+0000","last_became_peered":"2024-09-13T14:38:04.705748+0000","last_unstale":"2024-09-13T14:38:18.315620+0000","last_undegraded":"2024-09-13T14:38:18.315620+0000","last_fullsized":"2024-09-13T14:38:18.315620+0000","mapping_epoch":273,"log_start":"258'5446","ondisk_log_start":"258'5446","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173015040,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2640,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12270,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2640}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":398,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.576884+0000","last_change":"2024-09-13T14:38:03.566312+0000","last_active":"2024-09-13T14:38:06.576884+0000","last_peered":"2024-09-13T14:38:06.576884+0000","last_clean":"2024-09-13T14:38:06.576884+0000","last_became_active":"2024-09-13T14:38:03.565875+0000","last_became_peered":"2024-09-13T14:38:03.565875+0000","last_unstale":"2024-09-13T14:38:06.576884+0000","last_undegraded":"2024-09-13T14:38:06.576884+0000","last_fullsized":"2024-09-13T14:38:06.576884+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:05.346013+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":850,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.609087+0000","last_change":"2024-09-13T14:24:00.652797+0000","last_active":"2024-09-13T14:38:09.609087+0000","last_peered":"2024-09-13T14:38:09.609087+0000","last_clean":"2024-09-13T14:38:09.609087+0000","last_became_active":"2024-09-13T14:24:00.652187+0000","last_became_peered":"2024-09-13T14:24:00.652187+0000","last_unstale":"2024-09-13T14:38:09.609087+0000","last_undegraded":"2024-09-13T14:38:09.609087+0000","last_fullsized":"2024-09-13T14:38:09.609087+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:37.302252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":375,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.636409+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:38:09.636409+0000","last_peered":"2024-09-13T14:38:09.636409+0000","last_clean":"2024-09-13T14:38:09.636409+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:38:09.636409+0000","last_undegraded":"2024-09-13T14:38:09.636409+0000","last_fullsized":"2024-09-13T14:38:09.636409+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":426,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.609595+0000","last_change":"2024-09-13T14:23:27.723766+0000","last_active":"2024-09-13T14:38:09.609595+0000","last_peered":"2024-09-13T14:38:09.609595+0000","last_clean":"2024-09-13T14:38:09.609595+0000","last_became_active":"2024-09-13T14:23:27.722542+0000","last_became_peered":"2024-09-13T14:23:27.722542+0000","last_unstale":"2024-09-13T14:38:09.609595+0000","last_undegraded":"2024-09-13T14:38:09.609595+0000","last_fullsized":"2024-09-13T14:38:09.609595+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:06.107636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"277'12553","reported_seq":12591,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:22.960852+0000","last_change":"2024-09-13T14:34:57.522448+0000","last_active":"2024-09-13T14:38:22.960852+0000","last_peered":"2024-09-13T14:38:22.960852+0000","last_clean":"2024-09-13T14:38:22.960852+0000","last_became_active":"2024-09-13T14:34:57.522167+0000","last_became_peered":"2024-09-13T14:34:57.522167+0000","last_unstale":"2024-09-13T14:38:22.960852+0000","last_undegraded":"2024-09-13T14:38:22.960852+0000","last_fullsized":"2024-09-13T14:38:22.960852+0000","mapping_epoch":247,"log_start":"258'5797","ondisk_log_start":"258'5797","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:22:47.802147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163774464,"num_objects":2499,"num_object_clones":0,"num_object_copies":7497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12553,"num_write_kb":481664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":128,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592055+0000","last_change":"2024-09-13T14:30:40.695929+0000","last_active":"2024-09-13T14:38:06.592055+0000","last_peered":"2024-09-13T14:38:06.592055+0000","last_clean":"2024-09-13T14:38:06.592055+0000","last_became_active":"2024-09-13T14:30:11.460011+0000","last_became_peered":"2024-09-13T14:30:11.460011+0000","last_unstale":"2024-09-13T14:38:06.592055+0000","last_undegraded":"2024-09-13T14:38:06.592055+0000","last_fullsized":"2024-09-13T14:38:06.592055+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:48.663979+0000","scrub_duration":0.0025449719999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":908,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.591977+0000","last_change":"2024-09-13T14:34:46.566197+0000","last_active":"2024-09-13T14:38:06.591977+0000","last_peered":"2024-09-13T14:38:06.591977+0000","last_clean":"2024-09-13T14:38:06.591977+0000","last_became_active":"2024-09-13T14:34:46.565768+0000","last_became_peered":"2024-09-13T14:34:46.565768+0000","last_unstale":"2024-09-13T14:38:06.591977+0000","last_undegraded":"2024-09-13T14:38:06.591977+0000","last_fullsized":"2024-09-13T14:38:06.591977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:12:46.443713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":941,"num_read_kb":3168,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630835+0000","last_change":"2024-09-13T14:38:03.574313+0000","last_active":"2024-09-13T14:38:09.630835+0000","last_peered":"2024-09-13T14:38:09.630835+0000","last_clean":"2024-09-13T14:38:09.630835+0000","last_became_active":"2024-09-13T14:38:03.573500+0000","last_became_peered":"2024-09-13T14:38:03.573500+0000","last_unstale":"2024-09-13T14:38:09.630835+0000","last_undegraded":"2024-09-13T14:38:09.630835+0000","last_fullsized":"2024-09-13T14:38:09.630835+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:31.718072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":396,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634823+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:38:09.634823+0000","last_peered":"2024-09-13T14:38:09.634823+0000","last_clean":"2024-09-13T14:38:09.634823+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:38:09.634823+0000","last_undegraded":"2024-09-13T14:38:09.634823+0000","last_fullsized":"2024-09-13T14:38:09.634823+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"277'12343","reported_seq":14928,"reported_epoch":277,"state":"active+remapped+backfilling","last_fresh":"2024-09-13T14:38:20.237262+0000","last_change":"2024-09-13T14:38:04.889738+0000","last_active":"2024-09-13T14:38:20.237262+0000","last_peered":"2024-09-13T14:38:20.237262+0000","last_clean":"2024-09-13T14:38:03.090596+0000","last_became_active":"2024-09-13T14:38:04.666308+0000","last_became_peered":"2024-09-13T14:38:04.666308+0000","last_unstale":"2024-09-13T14:38:20.237262+0000","last_undegraded":"2024-09-13T14:38:20.237262+0000","last_fullsized":"2024-09-13T14:38:20.237262+0000","mapping_epoch":273,"log_start":"261'6445","ondisk_log_start":"261'6445","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166002688,"num_objects":2533,"num_object_clones":0,"num_object_copies":7599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1400,"num_objects_unfound":0,"num_objects_dirty":2533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12343,"num_write_kb":476032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1267,"num_bytes_recovered":83034112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2533}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":276,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582920+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:38:06.582920+0000","last_peered":"2024-09-13T14:38:06.582920+0000","last_clean":"2024-09-13T14:38:06.582920+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:38:06.582920+0000","last_undegraded":"2024-09-13T14:38:06.582920+0000","last_fullsized":"2024-09-13T14:38:06.582920+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":258,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592686+0000","last_change":"2024-09-13T14:24:00.652864+0000","last_active":"2024-09-13T14:38:06.592686+0000","last_peered":"2024-09-13T14:38:06.592686+0000","last_clean":"2024-09-13T14:38:06.592686+0000","last_became_active":"2024-09-13T14:24:00.652551+0000","last_became_peered":"2024-09-13T14:24:00.652551+0000","last_unstale":"2024-09-13T14:38:06.592686+0000","last_undegraded":"2024-09-13T14:38:06.592686+0000","last_fullsized":"2024-09-13T14:38:06.592686+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:33.959232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":975,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.635008+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:38:09.635008+0000","last_peered":"2024-09-13T14:38:09.635008+0000","last_clean":"2024-09-13T14:38:09.635008+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:38:09.635008+0000","last_undegraded":"2024-09-13T14:38:09.635008+0000","last_fullsized":"2024-09-13T14:38:09.635008+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":361,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.635286+0000","last_change":"2024-09-13T14:38:03.564945+0000","last_active":"2024-09-13T14:38:09.635286+0000","last_peered":"2024-09-13T14:38:09.635286+0000","last_clean":"2024-09-13T14:38:09.635286+0000","last_became_active":"2024-09-13T14:38:03.564058+0000","last_became_peered":"2024-09-13T14:38:03.564058+0000","last_unstale":"2024-09-13T14:38:09.635286+0000","last_undegraded":"2024-09-13T14:38:09.635286+0000","last_fullsized":"2024-09-13T14:38:09.635286+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:53.001161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"277'12232","reported_seq":12270,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:18.108575+0000","last_change":"2024-09-13T14:34:57.527395+0000","last_active":"2024-09-13T14:38:18.108575+0000","last_peered":"2024-09-13T14:38:18.108575+0000","last_clean":"2024-09-13T14:38:18.108575+0000","last_became_active":"2024-09-13T14:34:57.527133+0000","last_became_peered":"2024-09-13T14:34:57.527133+0000","last_unstale":"2024-09-13T14:38:18.108575+0000","last_undegraded":"2024-09-13T14:38:18.108575+0000","last_fullsized":"2024-09-13T14:38:18.108575+0000","mapping_epoch":247,"log_start":"254'3106","ondisk_log_start":"254'3106","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:16.255103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12232,"num_write_kb":477120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":276,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582533+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:38:06.582533+0000","last_peered":"2024-09-13T14:38:06.582533+0000","last_clean":"2024-09-13T14:38:06.582533+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:38:06.582533+0000","last_undegraded":"2024-09-13T14:38:06.582533+0000","last_fullsized":"2024-09-13T14:38:06.582533+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":375,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634919+0000","last_change":"2024-09-13T14:34:46.583932+0000","last_active":"2024-09-13T14:38:09.634919+0000","last_peered":"2024-09-13T14:38:09.634919+0000","last_clean":"2024-09-13T14:38:09.634919+0000","last_became_active":"2024-09-13T14:34:46.580751+0000","last_became_peered":"2024-09-13T14:34:46.580751+0000","last_unstale":"2024-09-13T14:38:09.634919+0000","last_undegraded":"2024-09-13T14:38:09.634919+0000","last_fullsized":"2024-09-13T14:38:09.634919+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:32:38.905892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":629,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.593771+0000","last_change":"2024-09-13T14:38:06.593527+0000","last_active":"2024-09-13T14:38:06.593771+0000","last_peered":"2024-09-13T14:38:06.593771+0000","last_clean":"2024-09-13T14:38:06.593771+0000","last_became_active":"2024-09-13T14:38:06.593161+0000","last_became_peered":"2024-09-13T14:38:06.593161+0000","last_unstale":"2024-09-13T14:38:06.593771+0000","last_undegraded":"2024-09-13T14:38:06.593771+0000","last_fullsized":"2024-09-13T14:38:06.593771+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:21.866277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":411,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631150+0000","last_change":"2024-09-13T14:21:39.298963+0000","last_active":"2024-09-13T14:38:09.631150+0000","last_peered":"2024-09-13T14:38:09.631150+0000","last_clean":"2024-09-13T14:38:09.631150+0000","last_became_active":"2024-09-13T14:21:39.298631+0000","last_became_peered":"2024-09-13T14:21:39.298631+0000","last_unstale":"2024-09-13T14:38:09.631150+0000","last_undegraded":"2024-09-13T14:38:09.631150+0000","last_fullsized":"2024-09-13T14:38:09.631150+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:41:45.654262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"277'12016","reported_seq":12095,"reported_epoch":277,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:38:20.199459+0000","last_change":"2024-09-13T14:38:04.865046+0000","last_active":"2024-09-13T14:38:20.199459+0000","last_peered":"2024-09-13T14:38:20.199459+0000","last_clean":"2024-09-13T14:38:03.105078+0000","last_became_active":"2024-09-13T14:38:04.677209+0000","last_became_peered":"2024-09-13T14:38:04.677209+0000","last_unstale":"2024-09-13T14:38:20.199459+0000","last_undegraded":"2024-09-13T14:38:20.199459+0000","last_fullsized":"2024-09-13T14:38:20.199459+0000","mapping_epoch":273,"log_start":"260'6035","ondisk_log_start":"260'6035","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":5981,"log_dups_size":0,"ondisk_log_size":5981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2660,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12016,"num_write_kb":469632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":2660}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":36,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:14.627163+0000","last_change":"2024-09-13T14:38:14.627163+0000","last_active":"2024-09-13T14:38:14.627163+0000","last_peered":"2024-09-13T14:38:14.627163+0000","last_clean":"2024-09-13T14:38:14.627163+0000","last_became_active":"2024-09-13T14:38:03.567057+0000","last_became_peered":"2024-09-13T14:38:03.567057+0000","last_unstale":"2024-09-13T14:38:14.627163+0000","last_undegraded":"2024-09-13T14:38:14.627163+0000","last_fullsized":"2024-09-13T14:38:14.627163+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:29:29.986898+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":422,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.638900+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:38:09.638900+0000","last_peered":"2024-09-13T14:38:09.638900+0000","last_clean":"2024-09-13T14:38:09.638900+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:38:09.638900+0000","last_undegraded":"2024-09-13T14:38:09.638900+0000","last_fullsized":"2024-09-13T14:38:09.638900+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":734,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.635114+0000","last_change":"2024-09-13T14:38:03.590316+0000","last_active":"2024-09-13T14:38:09.635114+0000","last_peered":"2024-09-13T14:38:09.635114+0000","last_clean":"2024-09-13T14:38:09.635114+0000","last_became_active":"2024-09-13T14:38:03.572606+0000","last_became_peered":"2024-09-13T14:38:03.572606+0000","last_unstale":"2024-09-13T14:38:09.635114+0000","last_undegraded":"2024-09-13T14:38:09.635114+0000","last_fullsized":"2024-09-13T14:38:09.635114+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:08.052165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":420,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631965+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:38:09.631965+0000","last_peered":"2024-09-13T14:38:09.631965+0000","last_clean":"2024-09-13T14:38:09.631965+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:38:09.631965+0000","last_undegraded":"2024-09-13T14:38:09.631965+0000","last_fullsized":"2024-09-13T14:38:09.631965+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"277'12347","reported_seq":12419,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:20.226250+0000","last_change":"2024-09-13T14:34:57.522594+0000","last_active":"2024-09-13T14:38:20.226250+0000","last_peered":"2024-09-13T14:38:20.226250+0000","last_clean":"2024-09-13T14:38:20.226250+0000","last_became_active":"2024-09-13T14:34:57.520039+0000","last_became_peered":"2024-09-13T14:34:57.520039+0000","last_unstale":"2024-09-13T14:38:20.226250+0000","last_undegraded":"2024-09-13T14:38:20.226250+0000","last_fullsized":"2024-09-13T14:38:20.226250+0000","mapping_epoch":247,"log_start":"260'6440","ondisk_log_start":"260'6440","created":247,"last_epoch_clean":248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:34:56.503172+0000","last_clean_scrub_stamp":"2024-09-13T14:34:56.503172+0000","objects_scrubbed":0,"log_size":5907,"log_dups_size":0,"ondisk_log_size":5907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:37.564876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12347,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":261,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634352+0000","last_change":"2024-09-13T14:30:40.516249+0000","last_active":"2024-09-13T14:38:09.634352+0000","last_peered":"2024-09-13T14:38:09.634352+0000","last_clean":"2024-09-13T14:38:09.634352+0000","last_became_active":"2024-09-13T14:30:11.437355+0000","last_became_peered":"2024-09-13T14:30:11.437355+0000","last_unstale":"2024-09-13T14:38:09.634352+0000","last_undegraded":"2024-09-13T14:38:09.634352+0000","last_fullsized":"2024-09-13T14:38:09.634352+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:46.175551+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":446,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630593+0000","last_change":"2024-09-13T14:38:03.598137+0000","last_active":"2024-09-13T14:38:09.630593+0000","last_peered":"2024-09-13T14:38:09.630593+0000","last_clean":"2024-09-13T14:38:09.630593+0000","last_became_active":"2024-09-13T14:38:03.573467+0000","last_became_peered":"2024-09-13T14:38:03.573467+0000","last_unstale":"2024-09-13T14:38:09.630593+0000","last_undegraded":"2024-09-13T14:38:09.630593+0000","last_fullsized":"2024-09-13T14:38:09.630593+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:45.000226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":695,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.583006+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:38:06.583006+0000","last_peered":"2024-09-13T14:38:06.583006+0000","last_clean":"2024-09-13T14:38:06.583006+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:38:06.583006+0000","last_undegraded":"2024-09-13T14:38:06.583006+0000","last_fullsized":"2024-09-13T14:38:06.583006+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592124+0000","last_change":"2024-09-13T14:38:03.561630+0000","last_active":"2024-09-13T14:38:06.592124+0000","last_peered":"2024-09-13T14:38:06.592124+0000","last_clean":"2024-09-13T14:38:06.592124+0000","last_became_active":"2024-09-13T14:38:03.561171+0000","last_became_peered":"2024-09-13T14:38:03.561171+0000","last_unstale":"2024-09-13T14:38:06.592124+0000","last_undegraded":"2024-09-13T14:38:06.592124+0000","last_fullsized":"2024-09-13T14:38:06.592124+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:22:40.119653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":234,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.582784+0000","last_change":"2024-09-13T14:38:03.573390+0000","last_active":"2024-09-13T14:38:06.582784+0000","last_peered":"2024-09-13T14:38:06.582784+0000","last_clean":"2024-09-13T14:38:06.582784+0000","last_became_active":"2024-09-13T14:38:03.573044+0000","last_became_peered":"2024-09-13T14:38:03.573044+0000","last_unstale":"2024-09-13T14:38:06.582784+0000","last_undegraded":"2024-09-13T14:38:06.582784+0000","last_fullsized":"2024-09-13T14:38:06.582784+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:24.522738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":77,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577113+0000","last_change":"2024-09-13T14:38:03.582168+0000","last_active":"2024-09-13T14:38:06.577113+0000","last_peered":"2024-09-13T14:38:06.577113+0000","last_clean":"2024-09-13T14:38:06.577113+0000","last_became_active":"2024-09-13T14:38:03.565923+0000","last_became_peered":"2024-09-13T14:38:03.565923+0000","last_unstale":"2024-09-13T14:38:06.577113+0000","last_undegraded":"2024-09-13T14:38:06.577113+0000","last_fullsized":"2024-09-13T14:38:06.577113+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:43.033859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":374,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.634782+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:38:09.634782+0000","last_peered":"2024-09-13T14:38:09.634782+0000","last_clean":"2024-09-13T14:38:09.634782+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:38:09.634782+0000","last_undegraded":"2024-09-13T14:38:09.634782+0000","last_fullsized":"2024-09-13T14:38:09.634782+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":260,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.577221+0000","last_change":"2024-09-13T14:38:03.620319+0000","last_active":"2024-09-13T14:38:06.577221+0000","last_peered":"2024-09-13T14:38:06.577221+0000","last_clean":"2024-09-13T14:38:06.577221+0000","last_became_active":"2024-09-13T14:38:03.569905+0000","last_became_peered":"2024-09-13T14:38:03.569905+0000","last_unstale":"2024-09-13T14:38:06.577221+0000","last_undegraded":"2024-09-13T14:38:06.577221+0000","last_fullsized":"2024-09-13T14:38:06.577221+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:31:18.469972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592473+0000","last_change":"2024-09-13T14:34:46.565021+0000","last_active":"2024-09-13T14:38:06.592473+0000","last_peered":"2024-09-13T14:38:06.592473+0000","last_clean":"2024-09-13T14:38:06.592473+0000","last_became_active":"2024-09-13T14:34:46.563784+0000","last_became_peered":"2024-09-13T14:34:46.563784+0000","last_unstale":"2024-09-13T14:38:06.592473+0000","last_undegraded":"2024-09-13T14:38:06.592473+0000","last_fullsized":"2024-09-13T14:38:06.592473+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:00.380107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":379,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630866+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:38:09.630866+0000","last_peered":"2024-09-13T14:38:09.630866+0000","last_clean":"2024-09-13T14:38:09.630866+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:38:09.630866+0000","last_undegraded":"2024-09-13T14:38:09.630866+0000","last_fullsized":"2024-09-13T14:38:09.630866+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":419,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631560+0000","last_change":"2024-09-13T14:24:00.652786+0000","last_active":"2024-09-13T14:38:09.631560+0000","last_peered":"2024-09-13T14:38:09.631560+0000","last_clean":"2024-09-13T14:38:09.631560+0000","last_became_active":"2024-09-13T14:24:00.652464+0000","last_became_peered":"2024-09-13T14:24:00.652464+0000","last_unstale":"2024-09-13T14:38:09.631560+0000","last_undegraded":"2024-09-13T14:38:09.631560+0000","last_fullsized":"2024-09-13T14:38:09.631560+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:59.681173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":435,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.646125+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:38:09.646125+0000","last_peered":"2024-09-13T14:38:09.646125+0000","last_clean":"2024-09-13T14:38:09.646125+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:38:09.646125+0000","last_undegraded":"2024-09-13T14:38:09.646125+0000","last_fullsized":"2024-09-13T14:38:09.646125+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":257,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.581792+0000","last_change":"2024-09-13T14:34:46.570197+0000","last_active":"2024-09-13T14:38:06.581792+0000","last_peered":"2024-09-13T14:38:06.581792+0000","last_clean":"2024-09-13T14:38:06.581792+0000","last_became_active":"2024-09-13T14:34:46.568152+0000","last_became_peered":"2024-09-13T14:34:46.568152+0000","last_unstale":"2024-09-13T14:38:06.581792+0000","last_undegraded":"2024-09-13T14:38:06.581792+0000","last_fullsized":"2024-09-13T14:38:06.581792+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:24:44.592857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":452,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.632040+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:38:09.632040+0000","last_peered":"2024-09-13T14:38:09.632040+0000","last_clean":"2024-09-13T14:38:09.632040+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:38:09.632040+0000","last_undegraded":"2024-09-13T14:38:09.632040+0000","last_fullsized":"2024-09-13T14:38:09.632040+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":422,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.630875+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:38:09.630875+0000","last_peered":"2024-09-13T14:38:09.630875+0000","last_clean":"2024-09-13T14:38:09.630875+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:38:09.630875+0000","last_undegraded":"2024-09-13T14:38:09.630875+0000","last_fullsized":"2024-09-13T14:38:09.630875+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":392,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.645030+0000","last_change":"2024-09-13T14:38:03.576101+0000","last_active":"2024-09-13T14:38:09.645030+0000","last_peered":"2024-09-13T14:38:09.645030+0000","last_clean":"2024-09-13T14:38:09.645030+0000","last_became_active":"2024-09-13T14:38:03.575676+0000","last_became_peered":"2024-09-13T14:38:03.575676+0000","last_unstale":"2024-09-13T14:38:09.645030+0000","last_undegraded":"2024-09-13T14:38:09.645030+0000","last_fullsized":"2024-09-13T14:38:09.645030+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:19.653722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":441,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.632947+0000","last_change":"2024-09-13T14:38:03.572344+0000","last_active":"2024-09-13T14:38:09.632947+0000","last_peered":"2024-09-13T14:38:09.632947+0000","last_clean":"2024-09-13T14:38:09.632947+0000","last_became_active":"2024-09-13T14:38:03.572024+0000","last_became_peered":"2024-09-13T14:38:03.572024+0000","last_unstale":"2024-09-13T14:38:09.632947+0000","last_undegraded":"2024-09-13T14:38:09.632947+0000","last_fullsized":"2024-09-13T14:38:09.632947+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:46.144811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":21,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.635376+0000","last_change":"2024-09-13T14:38:03.567837+0000","last_active":"2024-09-13T14:38:09.635376+0000","last_peered":"2024-09-13T14:38:09.635376+0000","last_clean":"2024-09-13T14:38:09.635376+0000","last_became_active":"2024-09-13T14:38:03.567250+0000","last_became_peered":"2024-09-13T14:38:03.567250+0000","last_unstale":"2024-09-13T14:38:09.635376+0000","last_undegraded":"2024-09-13T14:38:09.635376+0000","last_fullsized":"2024-09-13T14:38:09.635376+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:46.756604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":454,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.609294+0000","last_change":"2024-09-13T14:34:46.582553+0000","last_active":"2024-09-13T14:38:09.609294+0000","last_peered":"2024-09-13T14:38:09.609294+0000","last_clean":"2024-09-13T14:38:09.609294+0000","last_became_active":"2024-09-13T14:34:46.582146+0000","last_became_peered":"2024-09-13T14:34:46.582146+0000","last_unstale":"2024-09-13T14:38:09.609294+0000","last_undegraded":"2024-09-13T14:38:09.609294+0000","last_fullsized":"2024-09-13T14:38:09.609294+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:55:13.365210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":419,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631011+0000","last_change":"2024-09-13T14:38:03.574656+0000","last_active":"2024-09-13T14:38:09.631011+0000","last_peered":"2024-09-13T14:38:09.631011+0000","last_clean":"2024-09-13T14:38:09.631011+0000","last_became_active":"2024-09-13T14:38:03.573983+0000","last_became_peered":"2024-09-13T14:38:03.573983+0000","last_unstale":"2024-09-13T14:38:09.631011+0000","last_undegraded":"2024-09-13T14:38:09.631011+0000","last_fullsized":"2024-09-13T14:38:09.631011+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:45.794632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.636386+0000","last_change":"2024-09-13T14:38:03.577130+0000","last_active":"2024-09-13T14:38:09.636386+0000","last_peered":"2024-09-13T14:38:09.636386+0000","last_clean":"2024-09-13T14:38:09.636386+0000","last_became_active":"2024-09-13T14:38:03.576499+0000","last_became_peered":"2024-09-13T14:38:03.576499+0000","last_unstale":"2024-09-13T14:38:09.636386+0000","last_undegraded":"2024-09-13T14:38:09.636386+0000","last_fullsized":"2024-09-13T14:38:09.636386+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:15.156436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":274,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592616+0000","last_change":"2024-09-13T14:38:03.583875+0000","last_active":"2024-09-13T14:38:06.592616+0000","last_peered":"2024-09-13T14:38:06.592616+0000","last_clean":"2024-09-13T14:38:06.592616+0000","last_became_active":"2024-09-13T14:38:03.564988+0000","last_became_peered":"2024-09-13T14:38:03.564988+0000","last_unstale":"2024-09-13T14:38:06.592616+0000","last_undegraded":"2024-09-13T14:38:06.592616+0000","last_fullsized":"2024-09-13T14:38:06.592616+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:38.326087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.637267+0000","last_change":"2024-09-13T14:38:03.571640+0000","last_active":"2024-09-13T14:38:09.637267+0000","last_peered":"2024-09-13T14:38:09.637267+0000","last_clean":"2024-09-13T14:38:09.637267+0000","last_became_active":"2024-09-13T14:38:03.570363+0000","last_became_peered":"2024-09-13T14:38:03.570363+0000","last_unstale":"2024-09-13T14:38:09.637267+0000","last_undegraded":"2024-09-13T14:38:09.637267+0000","last_fullsized":"2024-09-13T14:38:09.637267+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:12.912639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":240,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.592692+0000","last_change":"2024-09-13T14:38:03.615992+0000","last_active":"2024-09-13T14:38:06.592692+0000","last_peered":"2024-09-13T14:38:06.592692+0000","last_clean":"2024-09-13T14:38:06.592692+0000","last_became_active":"2024-09-13T14:38:03.567120+0000","last_became_peered":"2024-09-13T14:38:03.567120+0000","last_unstale":"2024-09-13T14:38:06.592692+0000","last_undegraded":"2024-09-13T14:38:06.592692+0000","last_fullsized":"2024-09-13T14:38:06.592692+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:36.323998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":444,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631233+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:38:09.631233+0000","last_peered":"2024-09-13T14:38:09.631233+0000","last_clean":"2024-09-13T14:38:09.631233+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:38:09.631233+0000","last_undegraded":"2024-09-13T14:38:09.631233+0000","last_fullsized":"2024-09-13T14:38:09.631233+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":431,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631848+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:38:09.631848+0000","last_peered":"2024-09-13T14:38:09.631848+0000","last_clean":"2024-09-13T14:38:09.631848+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:38:09.631848+0000","last_undegraded":"2024-09-13T14:38:09.631848+0000","last_fullsized":"2024-09-13T14:38:09.631848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-09-13T14:38:06.581990+0000","last_change":"2024-09-13T14:38:03.562609+0000","last_active":"2024-09-13T14:38:06.581990+0000","last_peered":"2024-09-13T14:38:06.581990+0000","last_clean":"2024-09-13T14:38:06.581990+0000","last_became_active":"2024-09-13T14:38:03.562291+0000","last_became_peered":"2024-09-13T14:38:03.562291+0000","last_unstale":"2024-09-13T14:38:06.581990+0000","last_undegraded":"2024-09-13T14:38:06.581990+0000","last_fullsized":"2024-09-13T14:38:06.581990+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:15.128621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":425,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-09-13T14:38:09.631513+0000","last_change":"2024-09-13T14:23:27.721516+0000","last_active":"2024-09-13T14:38:09.631513+0000","last_peered":"2024-09-13T14:38:09.631513+0000","last_clean":"2024-09-13T14:38:09.631513+0000","last_became_active":"2024-09-13T14:23:27.721156+0000","last_became_peered":"2024-09-13T14:23:27.721156+0000","last_unstale":"2024-09-13T14:38:09.631513+0000","last_undegraded":"2024-09-13T14:38:09.631513+0000","last_fullsized":"2024-09-13T14:38:09.631513+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:47.440862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2719023128,"num_objects":41490,"num_object_clones":0,"num_object_copies":124470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9236,"num_objects_unfound":0,"num_objects_dirty":41490,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196774,"num_write_kb":7624385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4350,"num_bytes_recovered":285016064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8470081536,"data_stored":8470069320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111734,"ondisk_log_size":111734,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4890,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397826,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1283960,"kb_used_data":986608,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92461192,"statfs":{"total":95995035648,"available":94680260608,"internally_reserved":0,"allocated":1010286592,"data_stored":1008905349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594055,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101552,"kb_used_data":795816,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92643600,"statfs":{"total":95995035648,"available":94867046400,"internally_reserved":0,"allocated":814915584,"data_stored":813541897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790284,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004004,"kb_used_data":694172,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92741148,"statfs":{"total":95995035648,"available":94966935552,"internally_reserved":0,"allocated":710832128,"data_stored":709466978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":133143986512,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697070,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158820,"kb_used_data":1571036,"kb_used_omap":4,"kb_used_meta":587771,"kb_avail":91586332,"statfs":{"total":95995035648,"available":93784403968,"internally_reserved":0,"allocated":1608740864,"data_stored":1607375787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":601877579},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860597,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1489780,"kb_used_data":976812,"kb_used_omap":8,"kb_used_meta":512951,"kb_avail":92255372,"statfs":{"total":95995035648,"available":94469500928,"internally_reserved":0,"allocated":1000255488,"data_stored":998879293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":525262668},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020268,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232552,"kb_used_data":1088864,"kb_used_omap":9,"kb_used_meta":143670,"kb_avail":92512600,"statfs":{"total":95995035648,"available":94732902400,"internally_reserved":0,"allocated":1114996736,"data_stored":1113626994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":147118917},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187658,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1896616,"kb_used_data":1390176,"kb_used_omap":7,"kb_used_meta":506360,"kb_avail":91848536,"statfs":{"total":95995035648,"available":94052900864,"internally_reserved":0,"allocated":1423540224,"data_stored":1422170816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":518512971},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1406603264,"data_stored":1406599192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1099763712,"data_stored":1099759640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":992935936,"data_stored":992935936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1580072960,"data_stored":1580072960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":883818496,"data_stored":883818496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":702545920,"data_stored":702545920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":803344384,"data_stored":803340312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1000996864,"data_stored":1000996864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:38:32.208 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:38:32.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_2 pg_num 26 2024-09-13T14:38:32.209 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:32.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:32.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:32.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:32.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:32.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:32.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:32.646 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:32.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[103001]: pgmap v810: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 4938/107127 objects misplaced (4.609%); 25 MiB/s, 392 objects/s recovering 2024-09-13T14:38:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[107556]: pgmap v810: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 4938/107127 objects misplaced (4.609%); 25 MiB/s, 392 objects/s recovering 2024-09-13T14:38:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:32.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:32 smithi189 ceph-mon[90992]: pgmap v810: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 4938/107127 objects misplaced (4.609%); 25 MiB/s, 392 objects/s recovering 2024-09-13T14:38:32.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:32.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:32.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:32.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:32.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:33.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:33.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:33.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:33.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:33.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:33.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:33.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:33.444 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:38:33.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:33.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:33.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:33.627 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/319565189' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/319565189' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:33.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/319565189' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:38:33.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:33.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:33.910 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:33.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:33.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:34.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:34.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:34.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:34.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:34.238 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:38:34.238 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:38:34.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:34.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:34.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:34.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[107556]: pgmap v811: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4333/104205 objects misplaced (4.158%); 23 MiB/s, 374 objects/s recovering 2024-09-13T14:38:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:38:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:38:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[107556]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:38:34.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[103001]: pgmap v811: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4333/104205 objects misplaced (4.158%); 23 MiB/s, 374 objects/s recovering 2024-09-13T14:38:34.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:38:34.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:38:34.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:34.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:34 smithi017 ceph-mon[103001]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:38:34.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:34.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:34.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:34.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:34.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:34 smithi189 ceph-mon[90992]: pgmap v811: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 4333/104205 objects misplaced (4.158%); 23 MiB/s, 374 objects/s recovering 2024-09-13T14:38:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:34 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:38:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:34 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:38:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:38:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:34 smithi189 ceph-mon[90992]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:38:34.850 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:34.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:34.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:34.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:35.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:35.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:35.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:35.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:35.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:35.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:35.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:35.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:35.510 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:35.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:35.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:35.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:35.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[103001]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:38:35.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[107556]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:38:35.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:38:35.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:35.754 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:35 smithi189 ceph-mon[90992]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:38:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:35.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:35.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:35.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:35.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:35.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:36.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:36.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:38:36.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:36.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:36.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:36.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:36.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:36.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:36.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:36.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:36.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:36.662 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:36.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:36 smithi017 ceph-mon[103001]: pgmap v814: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3687/99954 objects misplaced (3.689%); 21 MiB/s, 330 objects/s recovering 2024-09-13T14:38:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:36 smithi017 ceph-mon[103001]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:38:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:36 smithi017 ceph-mon[107556]: pgmap v814: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3687/99954 objects misplaced (3.689%); 21 MiB/s, 330 objects/s recovering 2024-09-13T14:38:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:36 smithi017 ceph-mon[107556]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:38:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:36 smithi189 ceph-mon[90992]: pgmap v814: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 3687/99954 objects misplaced (3.689%); 21 MiB/s, 330 objects/s recovering 2024-09-13T14:38:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:36 smithi189 ceph-mon[90992]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:38:36.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:36.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:36.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:36.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:36.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:37.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:38:37.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:37.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:37.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:37.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:37.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:37.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:37.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:37] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-13T14:38:37.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:37.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:37.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:37.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:37.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:37 smithi189 ceph-mon[90992]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:38:37.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:37.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:37.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:37.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:38.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:38.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:37 smithi017 ceph-mon[103001]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:38:38.057 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:37 smithi017 ceph-mon[107556]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:38:38.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:38.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:38.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:38.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:38.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:38.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:38.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:38.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:38.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:38.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:38.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:38 smithi189 ceph-mon[90992]: pgmap v817: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 2699/96714 objects misplaced (2.791%); 20 MiB/s, 322 objects/s recovering 2024-09-13T14:38:38.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:38.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:38.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:38 smithi017 ceph-mon[103001]: pgmap v817: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 2699/96714 objects misplaced (2.791%); 20 MiB/s, 322 objects/s recovering 2024-09-13T14:38:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:38 smithi017 ceph-mon[107556]: pgmap v817: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 2699/96714 objects misplaced (2.791%); 20 MiB/s, 322 objects/s recovering 2024-09-13T14:38:38.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:39.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:39.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:39.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:39.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:39.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:39.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:39.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:39.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:39.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:39.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:39.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:39.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:39.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:39.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:39.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:39.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:39.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:39.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:39.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:40.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:40.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:40.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:40.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:40.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:40.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:40.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:40.480 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:40.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:40.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:40.621 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:40.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:40.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:40 smithi189 ceph-mon[90992]: pgmap v818: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 97 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 2016/92376 objects misplaced (2.182%); 22 MiB/s, 345 objects/s recovering 2024-09-13T14:38:40.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:40.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:40.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:40.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:40.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:40.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:40 smithi017 ceph-mon[103001]: pgmap v818: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 97 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 2016/92376 objects misplaced (2.182%); 22 MiB/s, 345 objects/s recovering 2024-09-13T14:38:40.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:40 smithi017 ceph-mon[107556]: pgmap v818: 99 pgs: 1 active+remapped+backfill_wait, 1 peering, 97 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 2016/92376 objects misplaced (2.182%); 22 MiB/s, 345 objects/s recovering 2024-09-13T14:38:41.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:41.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:41.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:41.209 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:41.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:41.337 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:41.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:41.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:41.505 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:41.520 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:41.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:41.647 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:41.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:41.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:41.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:41.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:41.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:41.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:42.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:42.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:42.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:42.237 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:42.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:42.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.364 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:42.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:42.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:42.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:42.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:42.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[103001]: pgmap v819: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 554/89427 objects misplaced (0.619%); 26 MiB/s, 417 objects/s recovering 2024-09-13T14:38:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[107556]: pgmap v819: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 554/89427 objects misplaced (0.619%); 26 MiB/s, 417 objects/s recovering 2024-09-13T14:38:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:42.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:42 smithi189 ceph-mon[90992]: pgmap v819: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 554/89427 objects misplaced (0.619%); 26 MiB/s, 417 objects/s recovering 2024-09-13T14:38:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:42.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:42.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:42.899 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:42.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:42.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:43.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:43.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:43.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:43.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:43.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:43.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:43.458 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:43.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:43.506 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 7 pg_num to 26 2024-09-13T14:38:43.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:43.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2507440290' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2507440290' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:43.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2507440290' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:38:43.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:43.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:43.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:43.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:44.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:44.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:44.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:44.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:44.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:44.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:44.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:44.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:44.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[103001]: pgmap v820: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 554/85620 objects misplaced (0.647%); 22 MiB/s, 347 objects/s recovering 2024-09-13T14:38:44.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:38:44.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[103001]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:38:44.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:44.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:44.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:44.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:44.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:44.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:44.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:44.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[107556]: pgmap v820: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 554/85620 objects misplaced (0.647%); 22 MiB/s, 347 objects/s recovering 2024-09-13T14:38:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:38:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:44 smithi017 ceph-mon[107556]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:38:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:44 smithi189 ceph-mon[90992]: pgmap v820: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 554/85620 objects misplaced (0.647%); 22 MiB/s, 347 objects/s recovering 2024-09-13T14:38:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:38:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:44 smithi189 ceph-mon[90992]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:38:45.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:45.055 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:45.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:45.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:45.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:45.303 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:45.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:45.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:45.530 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:45.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:45.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:45 smithi017 ceph-mon[103001]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:38:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:45 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:38:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:45 smithi017 ceph-mon[107556]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:38:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:45 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:38:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.724 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:45.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:45.825 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:45.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:45.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:45.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:45.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:45 smithi189 ceph-mon[90992]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:38:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:45 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:38:46.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:46.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:46.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:46.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:46.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:46.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:46.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:46.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:46.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:46.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:46.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:46.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:46.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:46.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:46.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:46.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:46.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:46.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:46.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[103001]: pgmap v823: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 554/80292 objects misplaced (0.690%); 16 MiB/s, 254 objects/s recovering 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[103001]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[107556]: pgmap v823: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 554/80292 objects misplaced (0.690%); 16 MiB/s, 254 objects/s recovering 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:38:46.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:46 smithi017 ceph-mon[107556]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:38:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:46 smithi189 ceph-mon[90992]: pgmap v823: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 554/80292 objects misplaced (0.690%); 16 MiB/s, 254 objects/s recovering 2024-09-13T14:38:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:46 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:38:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:46 smithi189 ceph-mon[90992]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:38:47.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:47.040 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:47.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:47.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:47.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:47.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:47.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:47.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:47.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:47.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:47.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:47] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-09-13T14:38:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[103001]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:38:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[103001]: 7.1 scrub starts 2024-09-13T14:38:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[103001]: 7.1 scrub ok 2024-09-13T14:38:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[103001]: 7.4 scrub starts 2024-09-13T14:38:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[107556]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:38:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[107556]: 7.1 scrub starts 2024-09-13T14:38:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[107556]: 7.1 scrub ok 2024-09-13T14:38:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:47 smithi017 ceph-mon[107556]: 7.4 scrub starts 2024-09-13T14:38:47.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:47.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:47.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:47.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:47.990 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:47.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:47 smithi189 ceph-mon[90992]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:38:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:47 smithi189 ceph-mon[90992]: 7.1 scrub starts 2024-09-13T14:38:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:47 smithi189 ceph-mon[90992]: 7.1 scrub ok 2024-09-13T14:38:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:47 smithi189 ceph-mon[90992]: 7.4 scrub starts 2024-09-13T14:38:48.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:48.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:48.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:48.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:48.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:48.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:48.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:48.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:38:48.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:38:48.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:38:48.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:38:48.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:38:48.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:38:48.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:38:48.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:38:48.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:38:48.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:48.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:48.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:48.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: pgmap v826: 109 pgs: 9 unknown, 1 peering, 99 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:38:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: 7.4 scrub ok 2024-09-13T14:38:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: 7.3 scrub starts 2024-09-13T14:38:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: 7.3 scrub ok 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/41631881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[107556]: 7.9 scrub starts 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: pgmap v826: 109 pgs: 9 unknown, 1 peering, 99 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: 7.4 scrub ok 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: 7.3 scrub starts 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: 7.3 scrub ok 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/41631881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:48.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:48 smithi017 ceph-mon[103001]: 7.9 scrub starts 2024-09-13T14:38:48.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:48.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:48.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:48.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:48.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:48.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: pgmap v826: 109 pgs: 9 unknown, 1 peering, 99 active+clean; 1.6 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 8.2 MiB/s, 131 objects/s recovering 2024-09-13T14:38:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: 7.4 scrub ok 2024-09-13T14:38:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: 7.3 scrub starts 2024-09-13T14:38:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: 7.3 scrub ok 2024-09-13T14:38:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/41631881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:38:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:48 smithi189 ceph-mon[90992]: 7.9 scrub starts 2024-09-13T14:38:49.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:49.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:49.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:49.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:49.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:49.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:49.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:49.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:49.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:49.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:49.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:49.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:49.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.8 scrub starts 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.8 scrub ok 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.9 scrub ok 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.5 scrub starts 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.2 scrub starts 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.2 scrub ok 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[103001]: 7.6 scrub starts 2024-09-13T14:38:49.813 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.8 scrub starts 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.8 scrub ok 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.9 scrub ok 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.5 scrub starts 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.2 scrub starts 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.2 scrub ok 2024-09-13T14:38:49.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:49 smithi017 ceph-mon[107556]: 7.6 scrub starts 2024-09-13T14:38:49.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:49.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:49.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:49.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.8 scrub starts 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.8 scrub ok 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.9 scrub ok 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.5 scrub starts 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.2 scrub starts 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.2 scrub ok 2024-09-13T14:38:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:49 smithi189 ceph-mon[90992]: 7.6 scrub starts 2024-09-13T14:38:50.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:50.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:50.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:50.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:50.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:217386 7.c e287/247 7:349735c0:::benchmark_data_smithi017_220393_object98252:head v 287'13171, mlcod=287'13169)", 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.117348+0000", 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012291931000000001, 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01232048, 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:38:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117348+0000", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117348+0000", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117349+0000", 2024-09-13T14:38:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2200000000000002e-07 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117355+0000", 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8220000000000003e-06 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117367+0000", 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2320999999999999e-05 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.117375+0000", 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7579999999999992e-06 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122448+0000", 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050732959999999997 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122471+0000", 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2623999999999999e-05 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:217388 7.f e287/247 7:f1277ccc:::benchmark_data_smithi017_220393_object98254:head v 287'13518, mlcod=287'13516)", 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.119413+0000", 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.010227333999999999, 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010309805999999999, 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119413+0000", 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119413+0000", 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119414+0000", 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5499999999999996e-07 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119420+0000", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7930000000000003e-06 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119429+0000", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2299999999999997e-06 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119435+0000", 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:38:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119505+0000", 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9591000000000005e-05 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.119523+0000", 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7907e-05 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:217387 7.7 e287/286 7:e26c1928:::benchmark_data_smithi017_220393_object98253:head v 287'13637, mlcod=287'13635)", 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.121730+0000", 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0079098429999999997, 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080413180000000004, 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:38:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121730+0000", 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121730+0000", 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121731+0000", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1600000000000001e-07 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121738+0000", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1679999999999998e-06 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121756+0000", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7903000000000001e-05 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121766+0000", 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6630000000000005e-06 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122017+0000", 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025128699999999999 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122038+0000", 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0905000000000001e-05 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:217392 7.c e287/247 7:346eb80f:::benchmark_data_smithi017_220393_object98258:head v 287'13172, mlcod=287'13169)", 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.121796+0000", 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0078439359999999993, 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080225190000000005, 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121796+0000", 2024-09-13T14:38:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121796+0000", 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121797+0000", 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8200000000000001e-07 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121801+0000", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2390000000000004e-06 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121809+0000", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9670000000000001e-06 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121814+0000", 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0289999999999996e-06 2024-09-13T14:38:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.126220+0000", 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044057480000000001 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.126240+0000", 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0534e-05 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:217390 7.11 7:8b60b01f:::benchmark_data_smithi017_220393_object98256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.121829+0000", 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0078112640000000001, 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080424969999999991, 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 217390 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121829+0000", 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121829+0000", 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121831+0000", 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.638e-06 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121835+0000", 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6229999999999996e-06 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121845+0000", 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3950000000000007e-06 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121851+0000", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3539999999999996e-06 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.121914+0000", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2817999999999994e-05 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122045+0000", 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130777 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.122351+0000", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030607900000000001 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125659+0000", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033086999999999999 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128656+0000", 2024-09-13T14:38:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002996964 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:217396 7.a 7:5b126021:::benchmark_data_smithi017_220393_object98262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.125762+0000", 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0038785489999999998, 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041879710000000004, 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:38:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 217396 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125762+0000", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125762+0000", 2024-09-13T14:38:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125764+0000", 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9549999999999999e-06 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125769+0000", 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3639999999999998e-06 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125809+0000", 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9742e-05 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125817+0000", 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5369999999999997e-06 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.125905+0000", 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7700000000000004e-05 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.126021+0000", 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116487 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.126147+0000", 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012544699999999999 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:217397 7.13 7.58ef7cd3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-09-13T14:38:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.127632+0000", 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0020082289999999998, 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023738629999999999, 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 217397 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127632+0000", 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127632+0000", 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127634+0000", 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.835e-06 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127639+0000", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8040000000000001e-06 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127656+0000", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7062999999999999e-05 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.127664+0000", 2024-09-13T14:38:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9319999999999996e-06 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45566.0:217395 7.f e287/247 7:f9203626:::benchmark_data_smithi017_220393_object98261:head v 287'13519, mlcod=287'13516)", 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.128764+0000", 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00087625300000000001, 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012888279999999999, 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:38:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128764+0000", 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128764+0000", 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128764+0000", 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9000000000000002e-07 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128769+0000", 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1969999999999998e-06 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128775+0000", 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7970000000000003e-06 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.128780+0000", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5959999999999997e-06 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129018+0000", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000238421 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129035+0000", 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6756999999999999e-05 2024-09-13T14:38:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45566.0:217396 7.a e287/247)", 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:50.129651+0000", 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9999886, 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044539599999999998, 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:38:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129651+0000", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129651+0000", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129652+0000", 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8000000000000005e-07 2024-09-13T14:38:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129656+0000", 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0199999999999996e-06 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129667+0000", 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1137e-05 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:38:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:50.129675+0000", 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2559999999999996e-06 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 9 2024-09-13T14:38:50.337 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:38:50.337 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:38:50.337 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:38:50.338 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-13T14:38:50.338 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:38:50.338 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:38:50.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-13T14:38:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:50.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:50.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:50.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:38:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:50.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:50.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:50.661 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:38:50.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:50.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:50.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.18 scrub starts 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.5 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.18 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.0 scrub starts 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: pgmap v827: 109 pgs: 3 unknown, 1 peering, 105 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.0 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.6 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.7 scrub starts 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.7 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.11 scrub starts 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.11 scrub ok 2024-09-13T14:38:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:50 smithi189 ceph-mon[90992]: 7.14 scrub starts 2024-09-13T14:38:50.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:50.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:50.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.18 scrub starts 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.5 scrub ok 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.18 scrub ok 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.0 scrub starts 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: pgmap v827: 109 pgs: 3 unknown, 1 peering, 105 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.0 scrub ok 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.6 scrub ok 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.7 scrub starts 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.7 scrub ok 2024-09-13T14:38:50.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.11 scrub starts 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.11 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[103001]: 7.14 scrub starts 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.18 scrub starts 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.5 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.18 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.0 scrub starts 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: pgmap v827: 109 pgs: 3 unknown, 1 peering, 105 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.0 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.6 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.7 scrub starts 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.7 scrub ok 2024-09-13T14:38:50.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.11 scrub starts 2024-09-13T14:38:50.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.11 scrub ok 2024-09-13T14:38:50.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:50 smithi017 ceph-mon[107556]: 7.14 scrub starts 2024-09-13T14:38:50.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:51.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:51.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:51.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:51.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:38:51.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:51.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:51.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:51.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:51.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:51.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:51.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:51.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:38:51.571 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:38:51.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:51.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:51.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:51.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:51.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: 7.14 scrub ok 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4064111284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: 7.12 scrub starts 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: 7.12 scrub ok 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:38:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[103001]: 7.19 scrub starts 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: 7.14 scrub ok 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4064111284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: 7.12 scrub starts 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: 7.12 scrub ok 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:38:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:51 smithi017 ceph-mon[107556]: 7.19 scrub starts 2024-09-13T14:38:51.955 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:51.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: 7.14 scrub ok 2024-09-13T14:38:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4064111284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:38:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: 7.12 scrub starts 2024-09-13T14:38:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: 7.12 scrub ok 2024-09-13T14:38:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:38:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:51 smithi189 ceph-mon[90992]: 7.19 scrub starts 2024-09-13T14:38:52.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:52.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:52.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:52.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:52.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:52.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:52.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:52.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.454 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:52.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:52.529 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:38:52.529 INFO:teuthology.orchestra.run.smithi189.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:38:52.529 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:38:52.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:52.577 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:52.679 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:52.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:52.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:52.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:52.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: 7.10 deep-scrub starts 2024-09-13T14:38:52.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: pgmap v828: 109 pgs: 109 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:52.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: 7.19 scrub ok 2024-09-13T14:38:52.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:52.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:38:52.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:38:52.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:38:52.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:38:52.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:52 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:38:52.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: 7.10 deep-scrub starts 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: pgmap v828: 109 pgs: 109 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: 7.19 scrub ok 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:38:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: 7.10 deep-scrub starts 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: pgmap v828: 109 pgs: 109 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: 7.19 scrub ok 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:38:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:52 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:38:52.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:52.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:53.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:53.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:53.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:53.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:53.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:53.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:53.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:53.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:53.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:53.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:53.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:53.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:53.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:53.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:53.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:53 smithi017 ceph-mon[103001]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:38:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:38:53.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:53 smithi017 ceph-mon[107556]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:38:53.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:38:53.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:53.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:53 smithi189 ceph-mon[90992]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:38:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:38:54.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:54.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:54.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:54.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:54.241 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:54.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:54.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:54.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:54.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:54.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:54.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: pgmap v831: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:54.619 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.620 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:54.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:54.631 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:54.721 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:54.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:54.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:54.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:54.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: pgmap v831: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:54.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:38:54.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: 7.10 starting backfill to osd.2 from (0'0,0'0] MAX to 287'13646 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: 7.10 starting backfill to osd.2 from (0'0,0'0] MAX to 287'13646 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:38:54.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:54 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:38:54.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:54.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:54.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: pgmap v831: 109 pgs: 109 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: 7.10 starting backfill to osd.2 from (0'0,0'0] MAX to 287'13646 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:38:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:54 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:38:55.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:55.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:55.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:55.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:55.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:55.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:55.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:55.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:55.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:55.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:55.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:55.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:55.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:55.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:55.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:55.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:55 smithi017 ceph-mon[103001]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:38:55.869 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:55 smithi017 ceph-mon[103001]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:38:55.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:55 smithi017 ceph-mon[107556]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:38:55.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:55 smithi017 ceph-mon[107556]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:38:55.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:55.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:55.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:55 smithi189 ceph-mon[90992]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:38:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:55 smithi189 ceph-mon[90992]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:38:56.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:56.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:56.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:38:56.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:56.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:56.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:56.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:56.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:56.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:56.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:56.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:56.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:56.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:56.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:56.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:56.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:56.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[103001]: pgmap v834: 109 pgs: 1 remapped+peering, 108 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[103001]: 7.11 starting backfill to osd.7 from (0'0,0'0] MAX to 288'13527 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[103001]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[107556]: pgmap v834: 109 pgs: 1 remapped+peering, 108 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[107556]: 7.11 starting backfill to osd.7 from (0'0,0'0] MAX to 288'13527 2024-09-13T14:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:56 smithi017 ceph-mon[107556]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:38:56.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:56.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:38:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:56 smithi189 ceph-mon[90992]: pgmap v834: 109 pgs: 1 remapped+peering, 108 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:56 smithi189 ceph-mon[90992]: 7.11 starting backfill to osd.7 from (0'0,0'0] MAX to 288'13527 2024-09-13T14:38:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:56 smithi189 ceph-mon[90992]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:38:57.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:57.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:57.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:57.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:57.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:57.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:57.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:38:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:38:57] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:38:57.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:38:57.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:38:57.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:57.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:57.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:57.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:57.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:57.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:58.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:58.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:58.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:58.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:58.310 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:38:58.310 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:38:58.310 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2024-09-13T14:38:58.310 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T14:38:58.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5 2024-09-13T14:38:58.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:58.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[103001]: pgmap v837: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:58.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:58.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[107556]: pgmap v837: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:58.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:58.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:58.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:58.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:58.505 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:38:58 smithi189 systemd[1]: Stopping Ceph osd.5 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-mon[90992]: pgmap v837: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-13T14:38:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:38:58.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:58.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:58.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:58.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:58.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:58.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:58.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:58.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:58.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:58.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:59.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[44832]: 2024-09-13T14:38:58.664+0000 7f88c6249640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:38:59.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[44832]: 2024-09-13T14:38:58.664+0000 7f88c6249640 -1 osd.5 294 *** Got signal Terminated *** 2024-09-13T14:38:59.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:38:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[44832]: 2024-09-13T14:38:58.664+0000 7f88c6249640 -1 osd.5 294 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:59.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:38:59.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:38:59.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:38:59.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:38:59.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:38:59.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:59.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:38:59.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[103001]: 7.10 scrub starts 2024-09-13T14:38:59.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[103001]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:38:59.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[103001]: osd.5 marked itself down and dead 2024-09-13T14:38:59.323 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[107556]: 7.10 scrub starts 2024-09-13T14:38:59.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[107556]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:38:59.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:38:59 smithi017 ceph-mon[107556]: osd.5 marked itself down and dead 2024-09-13T14:38:59.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:38:59.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:38:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:59 smithi189 ceph-mon[90992]: 7.10 scrub starts 2024-09-13T14:38:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:59 smithi189 ceph-mon[90992]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:38:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:38:59 smithi189 ceph-mon[90992]: osd.5 marked itself down and dead 2024-09-13T14:38:59.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:38:59.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:38:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:38:59.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:59.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:59.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:59.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:59.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:59.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:38:59.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:38:59.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:38:59.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:38:59.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:38:59.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:38:59.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:38:59.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:38:59.923 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:38:59.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:38:59.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:38:59.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:38:59.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:38:59.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:38:59.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:38:59.925 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:38:59.925 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:38:59.926 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:38:59.926 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:00.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:00.056 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:38:59 smithi189 podman[164126]: 2024-09-13 14:38:59.755162863 +0000 UTC m=+1.218981226 container died 768ac2ae3f9255e9433fe04021dadb8ab3622f7f0a80d32aa7b336644c532a36 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, ceph=True) 2024-09-13T14:39:00.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:00.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: pgmap v839: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 8.1 MiB/s, 131 objects/s recovering 2024-09-13T14:39:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: osdmap e295: 8 total, 7 up, 7 in 2024-09-13T14:39:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/368910965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[103001]: osdmap e296: 8 total, 7 up, 7 in 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: pgmap v839: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 8.1 MiB/s, 131 objects/s recovering 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: osdmap e295: 8 total, 7 up, 7 in 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/368910965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:00 smithi017 ceph-mon[107556]: osdmap e296: 8 total, 7 up, 7 in 2024-09-13T14:39:00.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:00.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:00.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:00.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:00 smithi189 podman[164126]: 2024-09-13 14:39:00.279712744 +0000 UTC m=+1.743531105 container cleanup 768ac2ae3f9255e9433fe04021dadb8ab3622f7f0a80d32aa7b336644c532a36 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T14:39:00.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:00 smithi189 bash[164126]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: pgmap v839: 109 pgs: 1 active+clean+remapped, 1 remapped+peering, 107 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 8.1 MiB/s, 131 objects/s recovering 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: osdmap e295: 8 total, 7 up, 7 in 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/368910965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:00.507 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:00 smithi189 ceph-mon[90992]: osdmap e296: 8 total, 7 up, 7 in 2024-09-13T14:39:00.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:00.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:00.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:00.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:39:00.736 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:39:00.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:00.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:00.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:00.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:01.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:01.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:01.005 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:00 smithi189 podman[164230]: 2024-09-13 14:39:00.623754842 +0000 UTC m=+0.864639376 container remove 768ac2ae3f9255e9433fe04021dadb8ab3622f7f0a80d32aa7b336644c532a36 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:39:01.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:01.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:01.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:39:01.222 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:39:01.222 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:01.222 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:01.222 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:01.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:01.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:01.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:01.303 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:00.927043192 +0000 UTC m=+0.018889120 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:39:01.304 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:01.050014896 +0000 UTC m=+0.141860821 container create c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T14:39:01.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:01 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:39:01.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:01 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:39:01.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:01.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:01.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:01.313 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:01.313 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:01.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:01.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:01.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:01.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:01.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:01.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:01.589 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:01.303015165 +0000 UTC m=+0.394861094 container init c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:39:01.589 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:01.309011425 +0000 UTC m=+0.400857344 container start c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:39:01.590 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:01.419774331 +0000 UTC m=+0.511620259 container attach c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T14:39:01.590 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:01 smithi189 podman[164335]: 2024-09-13 14:39:01.519987246 +0000 UTC m=+0.611833172 container died c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T14:39:01.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:01.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:01.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:01.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:01.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:01.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:01.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:01.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:02.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:02.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:02.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:02.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:02.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:02 smithi189 ceph-mon[90992]: pgmap v842: 109 pgs: 3 active+undersized, 7 peering, 9 stale+active+clean, 1 remapped+peering, 2 active+undersized+degraded, 87 active+clean; 1.0 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 969/49116 objects degraded (1.973%) 2024-09-13T14:39:02.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:02 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:02.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:02 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 969/49116 objects degraded (1.973%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:02.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:02.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[107556]: pgmap v842: 109 pgs: 3 active+undersized, 7 peering, 9 stale+active+clean, 1 remapped+peering, 2 active+undersized+degraded, 87 active+clean; 1.0 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 969/49116 objects degraded (1.973%) 2024-09-13T14:39:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 969/49116 objects degraded (1.973%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[103001]: pgmap v842: 109 pgs: 3 active+undersized, 7 peering, 9 stale+active+clean, 1 remapped+peering, 2 active+undersized+degraded, 87 active+clean; 1.0 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 969/49116 objects degraded (1.973%) 2024-09-13T14:39:02.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:02.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:02 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 969/49116 objects degraded (1.973%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:02.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:02.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:02.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:02.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:02.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:02.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T14:39:02.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:39:02.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:02.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:02.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:02.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:02.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:02.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:02.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:03.005 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:02 smithi189 podman[164448]: 2024-09-13 14:39:02.642739112 +0000 UTC m=+1.108266098 container remove c51fe40580bf34d1bccf5b46037092bd72c330129ba1a0543a636be517afbaf8 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T14:39:03.005 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:02 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T14:39:03.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:02 smithi189 systemd[1]: Stopped Ceph osd.5 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:39:03.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:39:02 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service: Consumed 16min 1.893s CPU time. 2024-09-13T14:39:03.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:03.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:03.059 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:39:03.059 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T14:39:03.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:03.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:03.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:03.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:03.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:03.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:03.536 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:03.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:03.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:03.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:39:03.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:03.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:03.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:03.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:03.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:03.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:04.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:04.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:04.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:04.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:04.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:04.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:04 smithi017 ceph-mon[107556]: pgmap v843: 109 pgs: 11 active+undersized, 7 peering, 4 stale+active+clean, 1 remapped+peering, 6 active+undersized+degraded, 80 active+clean; 944 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 18 KiB/s rd, 49 KiB/s wr, 628 op/s; 1825/45180 objects degraded (4.039%); 17 KiB/s, 0 objects/s recovering 2024-09-13T14:39:04.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:04 smithi017 ceph-mon[103001]: pgmap v843: 109 pgs: 11 active+undersized, 7 peering, 4 stale+active+clean, 1 remapped+peering, 6 active+undersized+degraded, 80 active+clean; 944 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 18 KiB/s rd, 49 KiB/s wr, 628 op/s; 1825/45180 objects degraded (4.039%); 17 KiB/s, 0 objects/s recovering 2024-09-13T14:39:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:04 smithi189 ceph-mon[90992]: pgmap v843: 109 pgs: 11 active+undersized, 7 peering, 4 stale+active+clean, 1 remapped+peering, 6 active+undersized+degraded, 80 active+clean; 944 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 18 KiB/s rd, 49 KiB/s wr, 628 op/s; 1825/45180 objects degraded (4.039%); 17 KiB/s, 0 objects/s recovering 2024-09-13T14:39:04.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:04.607 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:04.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:04.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:04.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:04.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:04.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:04.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:04.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:05.078 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:05.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:05.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:05.232 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:05.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:05.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:05 smithi017 ceph-mon[103001]: 7.10 scrub starts 2024-09-13T14:39:05.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:05 smithi017 ceph-mon[107556]: 7.10 scrub starts 2024-09-13T14:39:05.342 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:05.466 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:05.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:05 smithi189 ceph-mon[90992]: 7.10 scrub starts 2024-09-13T14:39:05.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:05.618 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:05.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:05.744 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:05.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:05.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:05.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:05.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:05.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:06.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:06.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:06.189 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:39:06.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:06.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:06.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[103001]: pgmap v844: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 815 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 25 KiB/s rd, 64 KiB/s wr, 751 op/s; 2807/38997 objects degraded (7.198%); 5.1 MiB/s, 80 objects/s recovering 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2140391544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[107556]: pgmap v844: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 815 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 25 KiB/s rd, 64 KiB/s wr, 751 op/s; 2807/38997 objects degraded (7.198%); 5.1 MiB/s, 80 objects/s recovering 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2140391544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:06 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:06.495 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:06 smithi189 ceph-mon[90992]: pgmap v844: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 815 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 25 KiB/s rd, 64 KiB/s wr, 751 op/s; 2807/38997 objects degraded (7.198%); 5.1 MiB/s, 80 objects/s recovering 2024-09-13T14:39:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:06 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T14:39:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2140391544' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:06 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:06.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:06.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:06.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:06.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:06.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:06.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:06.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:06.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:07.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:07.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.166 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:07.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:07.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:07.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:07.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[103001]: osdmap e297: 8 total, 7 up, 7 in 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:07.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-mon[107556]: osdmap e297: 8 total, 7 up, 7 in 2024-09-13T14:39:07.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:07.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:07 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:39:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:07 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:39:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:07 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:07.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:07 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:07.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:07 smithi189 ceph-mon[90992]: osdmap e297: 8 total, 7 up, 7 in 2024-09-13T14:39:07.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:07.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:07.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:07] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-09-13T14:39:07.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:07.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:07.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:07.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:07.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:08.060 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2024-09-13T14:39:08.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:39:08.060 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T14:39:08.060 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2024-09-13T14:39:08.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 0 2024-09-13T14:39:08.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:08.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:08.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:08.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:08.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:08.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:08.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[103001]: pgmap v846: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 681 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 KiB/s rd, 56 KiB/s wr, 818 op/s; 2084/32571 objects degraded (6.398%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:39:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 2084/32571 objects degraded (6.398%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[107556]: pgmap v846: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 681 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 KiB/s rd, 56 KiB/s wr, 818 op/s; 2084/32571 objects degraded (6.398%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:39:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 2084/32571 objects degraded (6.398%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:08 smithi189 ceph-mon[90992]: pgmap v846: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 681 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 22 KiB/s rd, 56 KiB/s wr, 818 op/s; 2084/32571 objects degraded (6.398%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:39:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:08 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 2084/32571 objects degraded (6.398%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:08.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:08.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:08.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:08.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:08.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:08.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:08.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:08.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:08.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:08.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:09.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:09.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:39:09.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:09.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:09.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:09.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:09.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:09.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:09.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:09.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:09.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:09.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:09.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:09.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:09.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:09.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:10.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:10.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:10.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:10.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:10.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:10.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:10.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:10.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:10 smithi017 ceph-mon[103001]: 7.10 scrub starts 2024-09-13T14:39:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:10 smithi017 ceph-mon[103001]: pgmap v847: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 495 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 23 KiB/s rd, 50 KiB/s wr, 1.04k op/s; 2084/23619 objects degraded (8.823%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:39:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:10 smithi017 ceph-mon[107556]: 7.10 scrub starts 2024-09-13T14:39:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:10 smithi017 ceph-mon[107556]: pgmap v847: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 495 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 23 KiB/s rd, 50 KiB/s wr, 1.04k op/s; 2084/23619 objects degraded (8.823%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:39:10.456 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:10 smithi189 ceph-mon[90992]: 7.10 scrub starts 2024-09-13T14:39:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:10 smithi189 ceph-mon[90992]: pgmap v847: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 495 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 23 KiB/s rd, 50 KiB/s wr, 1.04k op/s; 2084/23619 objects degraded (8.823%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:39:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:10.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:10.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:10.608 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:39:10.608 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:39:10.608 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:10.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:10.610 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:39:10.644 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:39:10.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:10.674 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:39:10.674 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:10.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:10.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:10.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:10.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:10.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:10.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:11.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:11.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:11.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:11.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:11.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:11.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:11.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:11.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:11.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/266486011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:11.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/266486011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:39:11.494 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:39:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/266486011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:11.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:11.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:11.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:11.690 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:11.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:11.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:11.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:11.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:11.939 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:11.939 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:11.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:11.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:12.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:12.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:12.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:12.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:12.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:12.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:39:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:39:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[103001]: pgmap v848: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 375 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 20 KiB/s rd, 44 KiB/s wr, 1.05k op/s; 1222/17874 objects degraded (6.837%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[107556]: pgmap v848: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 375 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 20 KiB/s rd, 44 KiB/s wr, 1.05k op/s; 1222/17874 objects degraded (6.837%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:12.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:12 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:39:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:12 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:39:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:12 smithi189 ceph-mon[90992]: pgmap v848: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 375 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 20 KiB/s rd, 44 KiB/s wr, 1.05k op/s; 1222/17874 objects degraded (6.837%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:12.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:12.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:12.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:12.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:12.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:12.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:12.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:12.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:13.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:13.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:13.195 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:13.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:13.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:13.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:13 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 881/12801 objects degraded (6.882%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:13.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:13 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 881/12801 objects degraded (6.882%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:13.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:13.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:13.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:13.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:39:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:13 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 881/12801 objects degraded (6.882%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:13.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:13.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:13.643 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:13.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:13.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:13.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:13.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:13.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:13.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:13.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:14.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:14.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:14.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:14.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 119131 objects 2024-09-13T14:39:14.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :153.622 2024-09-13T14:39:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:14.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:14.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:14.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:14.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:14.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:14 smithi189 ceph-mon[90992]: pgmap v849: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 269 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 6.8 KiB/s rd, 15 KiB/s wr, 1.08k op/s; 881/12801 objects degraded (6.882%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:14.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.578 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:14.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:14.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:14 smithi017 ceph-mon[103001]: pgmap v849: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 269 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 6.8 KiB/s rd, 15 KiB/s wr, 1.08k op/s; 881/12801 objects degraded (6.882%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:14.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:14 smithi017 ceph-mon[107556]: pgmap v849: 109 pgs: 20 active+undersized, 13 active+undersized+degraded, 76 active+clean; 269 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 6.8 KiB/s rd, 15 KiB/s wr, 1.08k op/s; 881/12801 objects degraded (6.882%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:39:14.663 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-13T14:39:14.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-13T14:39:14.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:14.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:14.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:14.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:14.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:14.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:14.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:15.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:15.100 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:15.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:15.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:15.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:15.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.347 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:15.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:15.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:15.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:15.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:15 smithi189 ceph-mon[90992]: 7.10 scrub starts 2024-09-13T14:39:15.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:15.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:15 smithi017 ceph-mon[103001]: 7.10 scrub starts 2024-09-13T14:39:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:15 smithi017 ceph-mon[107556]: 7.10 scrub starts 2024-09-13T14:39:15.706 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.729 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:15.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:15.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:15.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:15.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:15.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:15.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:15.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:16.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:16.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:16.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:16.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:16.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:16.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:16.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:16.466 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:16.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:16 smithi189 ceph-mon[90992]: pgmap v850: 109 pgs: 21 active+undersized, 12 active+undersized+degraded, 76 active+clean; 122 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 436/5745 objects degraded (7.589%) 2024-09-13T14:39:16.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:16.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:16.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:16.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:16 smithi017 ceph-mon[103001]: pgmap v850: 109 pgs: 21 active+undersized, 12 active+undersized+degraded, 76 active+clean; 122 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 436/5745 objects degraded (7.589%) 2024-09-13T14:39:16.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:16 smithi017 ceph-mon[107556]: pgmap v850: 109 pgs: 21 active+undersized, 12 active+undersized+degraded, 76 active+clean; 122 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 436/5745 objects degraded (7.589%) 2024-09-13T14:39:16.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:16.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:16.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:16.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:16.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:16.948 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:39:16.948 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:39:16.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:202836 6.11 6:8e6aa851:::benchmark_data_smithi017_185083_object78224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:33:42.449851+0000", 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 334.28876089599999, 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92197488500000002, 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.949 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 202836 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449851+0000", 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449851+0000", 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449852+0000", 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2929999999999999e-06 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449857+0000", 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1050000000000003e-06 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449867+0000", 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.876e-06 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449874+0000", 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0389999999999999e-06 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449938+0000", 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3788999999999995e-05 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:42.449947+0000", 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7360000000000007e-06 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.367380+0000", 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91743264400000002 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.367518+0000", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013872900000000001 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.367929+0000", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000411054 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.371651+0000", 2024-09-13T14:39:16.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037213820000000001 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.371694+0000", 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3486999999999999e-05 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.371725+0000", 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0757999999999999e-05 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.371766+0000", 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0886999999999997e-05 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:33:43.371826+0000", 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0105999999999997e-05 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.956 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218864 7.10 7:0bb0242d:::benchmark_data_smithi017_220393_object99730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.157236164, 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1054092660000001, 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218864 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581377+0000", 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.471e-06 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581382+0000", 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581394+0000", 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1482e-05 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581404+0000", 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0696999999999999e-05 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583298+0000", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001893175 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583308+0000", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9830000000000008e-06 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588552+0000", 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.00524463 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588568+0000", 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5375000000000001e-05 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.675967+0000", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0873995139999999 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676058+0000", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1031000000000003e-05 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686713+0000", 2024-09-13T14:39:16.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01065461 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686785+0000", 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2291000000000003e-05 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218870 7.10 7:0fbe9f56:::benchmark_data_smithi017_220393_object99736:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:39:16.964 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.157194527000001, 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1036150889999998, 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218870 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581418+0000", 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1099999999999996e-07 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581421+0000", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9409999999999999e-06 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581428+0000", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2250000000000003e-06 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581434+0000", 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.716e-06 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583376+0000", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001942163 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583385+0000", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8380000000000008e-06 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588598+0000", 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0052131259999999 2024-09-13T14:39:16.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588606+0000", 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6040000000000004e-06 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676303+0000", 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0876976759999999 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:39:16.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676368+0000", 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5038999999999999e-05 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684968+0000", 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0086000250000000007 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685033+0000", 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4225000000000003e-05 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218875 7.10 7:0fe1bba5:::benchmark_data_smithi017_220393_object99741:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.157169897999999, 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1039573599999999, 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218875 2024-09-13T14:39:16.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581443+0000", 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2399999999999999e-07 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581445+0000", 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7350000000000001e-06 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581452+0000", 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8709999999999999e-06 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581457+0000", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1529999999999996e-06 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583857+0000", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023997739999999999 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583866+0000", 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1630000000000002e-06 2024-09-13T14:39:16.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588632+0000", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0047655849999999 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588639+0000", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5739999999999998e-06 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676415+0000", 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0877759490000001 2024-09-13T14:39:16.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676484+0000", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9036999999999996e-05 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685350+0000", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088651589999999992 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685399+0000", 2024-09-13T14:39:16.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9935999999999998e-05 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218879 7.10 7:0ecc6a5d:::benchmark_data_smithi017_220393_object99745:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.157147772999998, 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1043238309999999, 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218879 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581465+0000", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1699999999999999e-07 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581467+0000", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2759999999999999e-06 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581473+0000", 2024-09-13T14:39:16.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8640000000000001e-06 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581478+0000", 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.972e-06 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583901+0000", 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024232759999999998 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583909+0000", 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6909999999999997e-06 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588666+0000", 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004757033 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588673+0000", 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1409999999999999e-06 2024-09-13T14:39:16.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.677835+0000", 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.089162218 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.677941+0000", 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105562 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.678107+0000", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166837 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684704+0000", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065968249999999997 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685439+0000", 2024-09-13T14:39:16.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073457100000000005 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685524+0000", 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5492000000000001e-05 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685575+0000", 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0821e-05 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685649+0000", 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4110999999999996e-05 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685788+0000", 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013872400000000001 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.986 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218930 7.10 7:0b6a92cc:::benchmark_data_smithi017_220393_object99796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.100011305999999, 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0476642279999999, 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.987 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218930 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638602+0000", 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.263e-06 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638605+0000", 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4400000000000001e-06 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638614+0000", 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0650000000000005e-06 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638620+0000", 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5289999999999999e-06 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638718+0000", 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7928000000000002e-05 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638727+0000", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5519999999999993e-06 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588699+0000", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94997155300000002 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588706+0000", 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.007e-06 2024-09-13T14:39:16.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679498+0000", 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.090792016 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679637+0000", 2024-09-13T14:39:16.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138744 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679919+0000", 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000282081 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684714+0000", 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047948169999999998 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685830+0000", 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00111637 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685863+0000", 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3269000000000003e-05 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686082+0000", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000218736 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686137+0000", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5451000000000002e-05 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686265+0000", 2024-09-13T14:39:16.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012740700000000001 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218931 7.10 7:084393c0:::benchmark_data_smithi017_220393_object99797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.099983562999999, 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0480426669999998, 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:16.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218931 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:39:16.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638629+0000", 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4999999999999998e-07 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638631+0000", 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0030000000000001e-06 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:16.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638637+0000", 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8680000000000004e-06 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638643+0000", 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9059999999999999e-06 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.639140+0000", 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049731000000000005 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:16.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.639148+0000", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.276e-06 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588732+0000", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94958390299999995 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588739+0000", 2024-09-13T14:39:17.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.097e-06 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.682893+0000", 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.094153873 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.682981+0000", 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7678999999999999e-05 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.683143+0000", 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016239100000000001 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685297+0000", 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021539329999999998 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686379+0000", 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001081568 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686435+0000", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6307000000000001e-05 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686488+0000", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2930999999999999e-05 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686544+0000", 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6208000000000001e-05 2024-09-13T14:39:17.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686671+0000", 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127064 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218937 7.10 7:0ef37e94:::benchmark_data_smithi017_220393_object99803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.095867512000002, 2024-09-13T14:39:17.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.059473412, 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218937 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:39:17.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642746+0000", 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.705e-06 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642752+0000", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642768+0000", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6464999999999999e-05 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642777+0000", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3639999999999999e-06 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642845+0000", 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8206000000000004e-05 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642854+0000", 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4089999999999992e-06 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588765+0000", 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94591096799999996 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588772+0000", 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0299999999999996e-06 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.687837+0000", 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.099064214 2024-09-13T14:39:17.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.687968+0000", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131218 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.688318+0000", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035054800000000001 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698755+0000", 2024-09-13T14:39:17.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010436259 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698865+0000", 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011072400000000001 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698919+0000", 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3233999999999999e-05 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702047+0000", 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031281019999999998 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702097+0000", 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0550000000000002e-05 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702218+0000", 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120721 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.012 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218940 7.10 7:0c1ab008:::benchmark_data_smithi017_220393_object99806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.09319511, 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0815908670000001, 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218940 2024-09-13T14:39:17.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645418+0000", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4300000000000001e-06 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645424+0000", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4650000000000002e-06 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645436+0000", 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2493999999999999e-05 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645444+0000", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.191e-06 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645579+0000", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134559 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645589+0000", 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009e-05 2024-09-13T14:39:17.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588798+0000", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94320918200000003 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588806+0000", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.690573+0000", 2024-09-13T14:39:17.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1017671490000001 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.690707+0000", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134289 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.691015+0000", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030820199999999999 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702329+0000", 2024-09-13T14:39:17.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011313969 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702380+0000", 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0553000000000002e-05 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707199+0000", 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004819215 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.726803+0000", 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019604223 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.726859+0000", 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5495999999999999e-05 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727008+0000", 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014901500000000001 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218943 7.10 7:0a0d9885:::benchmark_data_smithi017_220393_object99809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.092610132000001, 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0812243530000001, 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.021 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218943 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646003+0000", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4600000000000003e-07 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646007+0000", 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5120000000000002e-06 2024-09-13T14:39:17.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646018+0000", 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1014e-05 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646025+0000", 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9990000000000002e-06 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646721+0000", 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00069556399999999997 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646732+0000", 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.065e-05 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588830+0000", 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94209882 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588837+0000", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5270000000000004e-06 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693014+0000", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.104177489 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693155+0000", 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140335 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693470+0000", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000315357 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702428+0000", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089583709999999997 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702480+0000", 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1990999999999997e-05 2024-09-13T14:39:17.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707284+0000", 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048038930000000001 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727023+0000", 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019738240000000001 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727070+0000", 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7880000000000002e-05 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727226+0000", 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015576500000000001 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218979 7.10 7:0cbc6b69:::benchmark_data_smithi017_220393_object99845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:39:17.029 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.054218535, 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0430168480000002, 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218979 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684395+0000", 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1689999999999999e-06 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684398+0000", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.692e-06 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684411+0000", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3132999999999999e-05 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684418+0000", 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7159999999999997e-06 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684474+0000", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5420000000000001e-05 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684482+0000", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6109999999999994e-06 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588860+0000", 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90437778700000004 2024-09-13T14:39:17.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588867+0000", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8220000000000001e-06 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696172+0000", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1073055059999999 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696298+0000", 2024-09-13T14:39:17.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012602499999999999 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696758+0000", 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045934900000000001 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707243+0000", 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010485387000000001 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707325+0000", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2330000000000006e-05 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715552+0000", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082263270000000003 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727236+0000", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011683901999999999 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727283+0000", 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7753000000000002e-05 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727410+0000", 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126919 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219048 7.10 7:0d03c00d:::benchmark_data_smithi017_220393_object99914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.037 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.990193064, 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9802098940000001, 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219048 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748420+0000", 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3060000000000001e-06 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748425+0000", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3089999999999997e-06 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748435+0000", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0013999999999999e-05 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748441+0000", 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6379999999999998e-06 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748494+0000", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2748999999999999e-05 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748503+0000", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8999999999999995e-06 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588908+0000", 2024-09-13T14:39:17.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.84040472799999999 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588916+0000", 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8210000000000008e-06 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698952+0000", 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1100361009999999 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.699074+0000", 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012147899999999999 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.699357+0000", 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028307199999999999 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707368+0000", 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080108569999999997 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707411+0000", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3529999999999998e-05 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715711+0000", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082997000000000001 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727424+0000", 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011712382 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.728508+0000", 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001084443 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.728629+0000", 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120865 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.045 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219076 7.10 7:0e85a74e:::benchmark_data_smithi017_220393_object99942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.048 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.964860083000001, 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9579237699999998, 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219076 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773753+0000", 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5409999999999999e-06 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773761+0000", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.52e-06 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773774+0000", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3345e-05 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773782+0000", 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.174e-06 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773837+0000", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4138000000000002e-05 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773846+0000", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6290000000000005e-06 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588947+0000", 2024-09-13T14:39:17.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.81510079700000004 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588955+0000", 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2919999999999996e-06 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702527+0000", 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.11357124 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702670+0000", 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014383199999999999 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.703005+0000", 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00033464600000000002 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715416+0000", 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012411294 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715596+0000", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017949700000000001 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723195+0000", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075989910000000003 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731482+0000", 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082870700000000005 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731541+0000", 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8644999999999998e-05 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731676+0000", 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013511900000000001 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219090 7.10 7:095d1507:::benchmark_data_smithi017_220393_object99956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.951932567, 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.945184759, 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219090 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786681+0000", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5e-06 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786686+0000", 2024-09-13T14:39:17.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1089999999999997e-06 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786699+0000", 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3156000000000001e-05 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786707+0000", 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4959999999999999e-06 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786773+0000", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6443000000000001e-05 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786786+0000", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2452999999999999e-05 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588984+0000", 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.80219853399999996 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588992+0000", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8860000000000004e-06 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.704886+0000", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.115894175 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.705013+0000", 2024-09-13T14:39:17.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012722100000000001 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.705310+0000", 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029679199999999999 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715507+0000", 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010196828999999999 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715793+0000", 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028566000000000003 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723236+0000", 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074429479999999996 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731685+0000", 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084498100000000003 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731732+0000", 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.689e-05 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731864+0000", 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013185700000000001 2024-09-13T14:39:17.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219132 7.10 7:0a96437d:::benchmark_data_smithi017_220393_object99998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.897718994999998, 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8911681979999999, 2024-09-13T14:39:17.066 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219132 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840894+0000", 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.15e-06 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840899+0000", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.403e-06 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840912+0000", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2991e-05 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840919+0000", 2024-09-13T14:39:17.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1160000000000001e-06 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840978+0000", 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.982e-05 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840989+0000", 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0308e-05 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589054+0000", 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.74806535900000004 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589067+0000", 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2627e-05 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707442+0000", 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1183756840000001 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707564+0000", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121487 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707871+0000", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030680099999999999 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715654+0000", 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007783351 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715913+0000", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025875399999999998 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723276+0000", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073634030000000001 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731874+0000", 2024-09-13T14:39:17.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085975170000000007 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731928+0000", 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3974000000000002e-05 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732061+0000", 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013345300000000001 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219157 7.10 7:0de9bdb4:::benchmark_data_smithi017_220393_object100023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.855308490999999, 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8489351599999999, 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219157 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.075 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883305+0000", 2024-09-13T14:39:17.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1510000000000001e-06 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883309+0000", 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9790000000000004e-06 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883319+0000", 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0462000000000001e-05 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883324+0000", 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3619999999999997e-06 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883370+0000", 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5890999999999998e-05 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883377+0000", 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3110000000000002e-06 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589109+0000", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.70573227299999997 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589118+0000", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6109999999999994e-06 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.709675+0000", 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1205571809999999 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.709796+0000", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001208 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.710116+0000", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032086300000000001 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715850+0000", 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057340749999999999 2024-09-13T14:39:17.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723068+0000", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072171129999999998 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723317+0000", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024899399999999999 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732070+0000", 2024-09-13T14:39:17.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087533660000000003 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732115+0000", 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5197000000000001e-05 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732239+0000", 2024-09-13T14:39:17.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012353099999999999 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219172 7.10 7:0f8dfd6f:::benchmark_data_smithi017_220393_object100038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 24.814938203000001, 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8087410400000001, 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:39:17.083 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219172 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:39:17.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923675+0000", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0139999999999999e-06 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923679+0000", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3699999999999997e-06 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923690+0000", 2024-09-13T14:39:17.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0795000000000001e-05 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923695+0000", 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4299999999999997e-06 2024-09-13T14:39:17.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923746+0000", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0368000000000003e-05 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923754+0000", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0320000000000003e-06 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589297+0000", 2024-09-13T14:39:17.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.66554344600000004 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589307+0000", 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0094e-05 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.712072+0000", 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.122764936 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.712972+0000", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00089972299999999995 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.713337+0000", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036460799999999999 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715969+0000", 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026328160000000001 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723112+0000", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071422380000000004 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723357+0000", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024494000000000002 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732247+0000", 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088901480000000005 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732291+0000", 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3951e-05 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732415+0000", 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012413100000000001 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:39:17.091 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:39:17.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:17.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:17.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:17.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:17.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:17.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:17.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:17.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:17.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:17 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:39:17.508 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:17] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.51.0" 2024-09-13T14:39:17.508 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:17 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:39:17.509 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:17 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:39:17.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:17.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:17.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:17.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:17.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:17.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:17.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:17.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:18.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:18.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:18.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:18.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:18.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:18.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:18.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:18.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:18.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:18 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:39:18.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:18 smithi017 ceph-mon[103001]: pgmap v851: 109 pgs: 23 active+undersized, 10 active+undersized+degraded, 76 active+clean; 37 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13/1635 objects degraded (0.795%) 2024-09-13T14:39:18.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:18 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:39:18.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:18 smithi017 ceph-mon[107556]: pgmap v851: 109 pgs: 23 active+undersized, 10 active+undersized+degraded, 76 active+clean; 37 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13/1635 objects degraded (0.795%) 2024-09-13T14:39:18.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:18.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:18.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:18.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:18.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:18 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:39:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:18 smithi189 ceph-mon[90992]: pgmap v851: 109 pgs: 23 active+undersized, 10 active+undersized+degraded, 76 active+clean; 37 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 13/1635 objects degraded (0.795%) 2024-09-13T14:39:18.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:18.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:39:18.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:18.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:18.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:18.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:19.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:19.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:19.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:19.132 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:19.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:19.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:19.260 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:39:19.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:19.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:19.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:19.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:19 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:39:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 13/1635 objects degraded (0.795%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:19.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:19.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:19 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:39:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 13/1635 objects degraded (0.795%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:19 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:39:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 13/1635 objects degraded (0.795%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:19.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:19.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:19.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:19.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:19.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:19.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:19.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:20.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:20.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:20.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:20.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:20.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:20.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:20.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:20.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:20.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.372 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:20.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: pgmap v852: 109 pgs: 1 active+clean+scrubbing, 23 active+undersized, 10 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: osdmap e298: 8 total, 7 up, 7 in 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: 7.15 scrub starts 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[107556]: 7.15 scrub ok 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: pgmap v852: 109 pgs: 1 active+clean+scrubbing, 23 active+undersized, 10 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: osdmap e298: 8 total, 7 up, 7 in 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: 7.15 scrub starts 2024-09-13T14:39:20.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:20 smithi017 ceph-mon[103001]: 7.15 scrub ok 2024-09-13T14:39:20.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:20.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:20.534 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:20.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: pgmap v852: 109 pgs: 1 active+clean+scrubbing, 23 active+undersized, 10 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:20.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:20.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2980988375' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:39:20.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: osdmap e298: 8 total, 7 up, 7 in 2024-09-13T14:39:20.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: 7.15 scrub starts 2024-09-13T14:39:20.538 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:20 smithi189 ceph-mon[90992]: 7.15 scrub ok 2024-09-13T14:39:20.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:20.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:20.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:20.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:20.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:20.763 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:39:20.763 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:39:20.763 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:39:20.763 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:39:20.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:20.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:20.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:20.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:21.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:21.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:21.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:21.227 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:21.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:21.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:21.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:21.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:21.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:21 smithi189 ceph-mon[90992]: 7.10 scrub starts 2024-09-13T14:39:21.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:21 smithi189 ceph-mon[90992]: 7.10 scrub ok 2024-09-13T14:39:21.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:21.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:39:21.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:21.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:21 smithi017 ceph-mon[103001]: 7.10 scrub starts 2024-09-13T14:39:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:21 smithi017 ceph-mon[103001]: 7.10 scrub ok 2024-09-13T14:39:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:21 smithi017 ceph-mon[107556]: 7.10 scrub starts 2024-09-13T14:39:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:21 smithi017 ceph-mon[107556]: 7.10 scrub ok 2024-09-13T14:39:21.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:21.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:21.816 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:21.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:21.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:21.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:21.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:22.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:22.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:22.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:22.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:22.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:22.267 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 0 (802) 2024-09-13T14:39:22.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:22.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:22.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[103001]: pgmap v854: 109 pgs: 1 active+clean+scrubbing, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/225 objects degraded (5.778%) 2024-09-13T14:39:22.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[103001]: 7.16 scrub starts 2024-09-13T14:39:22.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[103001]: 7.16 scrub ok 2024-09-13T14:39:22.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/39560325' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[107556]: pgmap v854: 109 pgs: 1 active+clean+scrubbing, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/225 objects degraded (5.778%) 2024-09-13T14:39:22.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[107556]: 7.16 scrub starts 2024-09-13T14:39:22.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[107556]: 7.16 scrub ok 2024-09-13T14:39:22.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/39560325' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:22.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:39:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:22 smithi189 ceph-mon[90992]: pgmap v854: 109 pgs: 1 active+clean+scrubbing, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/225 objects degraded (5.778%) 2024-09-13T14:39:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:22 smithi189 ceph-mon[90992]: 7.16 scrub starts 2024-09-13T14:39:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:22 smithi189 ceph-mon[90992]: 7.16 scrub ok 2024-09-13T14:39:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/39560325' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-13T14:39:22.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:22.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:22.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:22.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:22.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:22.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:22.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:22.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:22.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:22.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:23.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:23.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:23.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:23.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:23.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:23.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[103001]: 7.13 scrub starts 2024-09-13T14:39:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[103001]: 7.13 scrub ok 2024-09-13T14:39:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-13T14:39:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[103001]: osdmap e299: 8 total, 7 up, 7 in 2024-09-13T14:39:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[107556]: 7.13 scrub starts 2024-09-13T14:39:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[107556]: 7.13 scrub ok 2024-09-13T14:39:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-13T14:39:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:23 smithi017 ceph-mon[107556]: osdmap e299: 8 total, 7 up, 7 in 2024-09-13T14:39:23.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:23.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:23.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:23 smithi189 ceph-mon[90992]: 7.13 scrub starts 2024-09-13T14:39:23.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:23 smithi189 ceph-mon[90992]: 7.13 scrub ok 2024-09-13T14:39:23.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-13T14:39:23.540 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:23 smithi189 ceph-mon[90992]: osdmap e299: 8 total, 7 up, 7 in 2024-09-13T14:39:23.545 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:39:23.545 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:39:23.545 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:39:23.545 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:39:23.545 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:39:23.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:23.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:23.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:23.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:23.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:23.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:23.874 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:23.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:23.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:23.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:24.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:39:24.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:24.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:24.206 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:24.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:24.344 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:24.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:24.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: pgmap v856: 109 pgs: 3 peering, 1 active+clean+scrubbing, 21 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: osdmap e300: 8 total, 7 up, 7 in 2024-09-13T14:39:24.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 13/225 objects degraded (5.778%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:24.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: 7.17 scrub starts 2024-09-13T14:39:24.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[107556]: 7.17 scrub ok 2024-09-13T14:39:24.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: pgmap v856: 109 pgs: 3 peering, 1 active+clean+scrubbing, 21 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:24.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:24.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: osdmap e300: 8 total, 7 up, 7 in 2024-09-13T14:39:24.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 13/225 objects degraded (5.778%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:24.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: 7.17 scrub starts 2024-09-13T14:39:24.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:24 smithi017 ceph-mon[103001]: 7.17 scrub ok 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: pgmap v856: 109 pgs: 3 peering, 1 active+clean+scrubbing, 21 active+undersized, 9 active+undersized+degraded, 75 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s; 13/225 objects degraded (5.778%) 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: osdmap e300: 8 total, 7 up, 7 in 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 13/225 objects degraded (5.778%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: 7.17 scrub starts 2024-09-13T14:39:24.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:24 smithi189 ceph-mon[90992]: 7.17 scrub ok 2024-09-13T14:39:24.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:24.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:24.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:24.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:24.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:24.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:24.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:24.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:24.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:24.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:24.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:24.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:24.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:25.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:25.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:25.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:25.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:25.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3616702665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:25.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3616702665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:39:25.327 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:39:25.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:25.341 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:39:25.341 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:25.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:25.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:25.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3616702665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:25.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:25.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:25.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.624 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:25.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:25.725 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:25.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:25.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:25.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:25.939 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:25.939 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:25.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:25.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:25.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:26.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:26.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:26.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:26.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:26.284 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' removed 2024-09-13T14:39:26.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:26.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:26 smithi017 ceph-mon[103001]: pgmap v858: 109 pgs: 7 peering, 21 active+undersized, 8 active+undersized+degraded, 73 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/225 objects degraded (5.333%) 2024-09-13T14:39:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:26 smithi017 ceph-mon[107556]: pgmap v858: 109 pgs: 7 peering, 21 active+undersized, 8 active+undersized+degraded, 73 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/225 objects degraded (5.333%) 2024-09-13T14:39:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:26.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:26.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:26.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:26.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:26.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:26.714 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:26 smithi189 ceph-mon[90992]: pgmap v858: 109 pgs: 7 peering, 21 active+undersized, 8 active+undersized+degraded, 73 active+clean; 7.4 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/225 objects degraded (5.333%) 2024-09-13T14:39:26.715 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:26.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:26.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:39:26.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:26.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:26.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:26.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:26.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:26.999 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:27.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:27.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:27.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:39:27.158 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:39:27.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:27.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:27.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:27.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:27.375 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:27 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:27 smithi189 ceph-mon[90992]: osdmap e301: 8 total, 7 up, 7 in 2024-09-13T14:39:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:27.376 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:27] "GET /metrics HTTP/1.1" 200 34315 "" "Prometheus/2.51.0" 2024-09-13T14:39:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[103001]: osdmap e301: 8 total, 7 up, 7 in 2024-09-13T14:39:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3153536476' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[107556]: osdmap e301: 8 total, 7 up, 7 in 2024-09-13T14:39:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:27.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:27.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:27.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:27.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:27.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:27.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:27.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:27.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:27.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:27.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:28.001 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:39:28.001 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:39:28.002 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-13T14:39:28.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-13T14:39:28.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:28.166 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2024-09-13T14:39:28.166 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:39:28.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:28.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:28.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:28.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:28.443 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:28 smithi189 ceph-mon[90992]: pgmap v860: 83 pgs: 5 peering, 17 active+undersized, 8 active+undersized+degraded, 53 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/84 objects degraded (14.286%) 2024-09-13T14:39:28.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:28.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:28.515 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:28.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:28.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:28 smithi017 ceph-mon[103001]: pgmap v860: 83 pgs: 5 peering, 17 active+undersized, 8 active+undersized+degraded, 53 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/84 objects degraded (14.286%) 2024-09-13T14:39:28.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:28 smithi017 ceph-mon[107556]: pgmap v860: 83 pgs: 5 peering, 17 active+undersized, 8 active+undersized+degraded, 53 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 11 KiB/s rd, 4 op/s; 12/84 objects degraded (14.286%) 2024-09-13T14:39:28.679 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:28.680 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:28.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:28.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:28.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:28.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:28.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:28.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:28.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:29.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:29.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:29.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:39:29.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:29.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:29.303 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:39:29.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:29.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:29.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:39:29.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:29.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:29.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:29.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:29.659 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[107556]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/496930389' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[103001]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/496930389' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:29.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:29 smithi189 ceph-mon[90992]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 12/84 objects degraded (14.286%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/496930389' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:29.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:29.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:29.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:29.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:30.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:30.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:30.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:30.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:30.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:30.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[103001]: pgmap v861: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.8 KiB/s rd, 3 op/s; 13/84 objects degraded (15.476%) 2024-09-13T14:39:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-09-13T14:39:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[103001]: osdmap e302: 8 total, 7 up, 7 in 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[107556]: pgmap v861: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.8 KiB/s rd, 3 op/s; 13/84 objects degraded (15.476%) 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:30 smithi017 ceph-mon[107556]: osdmap e302: 8 total, 7 up, 7 in 2024-09-13T14:39:30.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:30.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-mon[90992]: pgmap v861: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 9.8 KiB/s rd, 3 op/s; 13/84 objects degraded (15.476%) 2024-09-13T14:39:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-09-13T14:39:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:39:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-mon[90992]: osdmap e302: 8 total, 7 up, 7 in 2024-09-13T14:39:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:30.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:30.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:30.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:30.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:30.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:30.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:30.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:30.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:30.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:31.059 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:31.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:31.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:31.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:31.307 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' created 2024-09-13T14:39:31.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:31.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:31.407 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:31.407 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:31.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:39:31.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:31.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:31.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:31.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:31.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:39:31.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:39:31.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:31.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:31.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:31.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:31.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:31.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:31.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:32.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:39:32.089 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:32.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:32.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:32.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:32.285 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:32.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:32.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[103001]: pgmap v863: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:39:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[103001]: osdmap e303: 8 total, 7 up, 7 in 2024-09-13T14:39:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[107556]: pgmap v863: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:39:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:32 smithi017 ceph-mon[107556]: osdmap e303: 8 total, 7 up, 7 in 2024-09-13T14:39:32.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:32.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:32.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:32.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:32 smithi189 ceph-mon[90992]: pgmap v863: 83 pgs: 20 active+undersized, 9 active+undersized+degraded, 54 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:32.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:32 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3696622094' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:39:32.606 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:32 smithi189 ceph-mon[90992]: osdmap e303: 8 total, 7 up, 7 in 2024-09-13T14:39:32.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:32.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:32.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:32.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:32.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:32.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:32.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:32.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:39:32.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:32.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:32.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:33.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:33.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:33.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-13T14:39:33.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:33.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:33.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:33.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:33.388 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:39:33.388 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:33.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:39:33.409 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:33.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:39:33.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:33.450 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:33.457 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:39:33.458 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:33.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:33.476 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:39:33.477 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:33.477 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:39:33.486 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:33.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:33.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:33.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:39:33.494 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:33.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:33 smithi189 ceph-mon[90992]: osdmap e304: 8 total, 7 up, 7 in 2024-09-13T14:39:33.511 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:39:33.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:33.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:33 smithi017 ceph-mon[103001]: osdmap e304: 8 total, 7 up, 7 in 2024-09-13T14:39:33.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:33 smithi017 ceph-mon[107556]: osdmap e304: 8 total, 7 up, 7 in 2024-09-13T14:39:33.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:33.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:33.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:33.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:33.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:33.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:33.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:33.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:33.987 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:33.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:33.992 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:189919 7.d 7:b52b85e9:::benchmark_data_smithi017_220393_object70785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:10.433400+0000", 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 83.557225232999997, 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.126719781, 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:33.993 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 189919 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433400+0000", 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:33.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433400+0000", 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433399+0000", 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433405+0000", 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4620000000000001e-06 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433418+0000", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2439000000000001e-05 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433424+0000", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7289999999999997e-06 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433466+0000", 2024-09-13T14:39:33.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2151999999999999e-05 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433593+0000", 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012762099999999999 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.433711+0000", 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117985 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:33.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.437020+0000", 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003308437 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.438349+0000", 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0013290870000000001 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.560058+0000", 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.121709186 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.560094+0000", 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5496e-05 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:10.560120+0000", 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6627000000000001e-05 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:33.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190271 7.d 7:bc6c9de1:::benchmark_data_smithi017_220393_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 82.953823205000006, 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33251491900000002, 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.000 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190271 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.001 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036801+0000", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036809+0000", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8369999999999997e-06 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036827+0000", 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8233999999999999e-05 2024-09-13T14:39:34.002 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036835+0000", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036859+0000", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3932999999999999e-05 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036943+0000", 2024-09-13T14:39:34.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4275000000000002e-05 2024-09-13T14:39:34.004 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.004 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.004 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.037020+0000", 2024-09-13T14:39:34.004 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6403999999999996e-05 2024-09-13T14:39:34.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.039717+0000", 2024-09-13T14:39:34.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026977440000000002 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045560+0000", 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058428489999999998 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369245+0000", 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32368499099999998 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369277+0000", 2024-09-13T14:39:34.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1498999999999998e-05 2024-09-13T14:39:34.007 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369317+0000", 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0759000000000001e-05 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190282 7.d 7:bc4d0eee:::benchmark_data_smithi017_220393_object71148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 82.944816704999994, 2024-09-13T14:39:34.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32360401599999999, 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190282 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045808+0000", 2024-09-13T14:39:34.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045813+0000", 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4319999999999998e-06 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045831+0000", 2024-09-13T14:39:34.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7847e-05 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045837+0000", 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3990000000000001e-06 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045875+0000", 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7481000000000002e-05 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045984+0000", 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108773 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.046540+0000", 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00055675799999999997 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.050112+0000", 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035716860000000001 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.055205+0000", 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0050931359999999998 2024-09-13T14:39:34.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369356+0000", 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31415043199999998 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369382+0000", 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6463000000000001e-05 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369413+0000", 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0812999999999998e-05 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.017 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190303 7.d 7:b1faf55c:::benchmark_data_smithi017_220393_object71169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.018 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:39:34.018 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 82.908460562000002, 2024-09-13T14:39:34.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28739342899999998, 2024-09-13T14:39:34.018 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.018 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190303 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.019 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082164+0000", 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082170+0000", 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2580000000000001e-06 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082183+0000", 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3101e-05 2024-09-13T14:39:34.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082190+0000", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9580000000000001e-06 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082232+0000", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2157000000000003e-05 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082315+0000", 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2902e-05 2024-09-13T14:39:34.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082401+0000", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5847999999999995e-05 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085041+0000", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026398659999999998 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085709+0000", 2024-09-13T14:39:34.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00066763599999999999 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369484+0000", 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28377533599999999 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369521+0000", 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7574999999999999e-05 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369559+0000", 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7100000000000001e-05 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195365 7.7 7:e23819ca:::benchmark_data_smithi017_220393_object76231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 75.674907425000001, 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.15216712900000001, 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.025 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195365 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.028 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315717+0000", 2024-09-13T14:39:34.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315726+0000", 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8640000000000008e-06 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315738+0000", 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1396999999999999e-05 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315743+0000", 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5969999999999999e-06 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315786+0000", 2024-09-13T14:39:34.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2601000000000001e-05 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315917+0000", 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000131326 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.316048+0000", 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013117099999999999 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.319756+0000", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003707307 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.428425+0000", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.108669617 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467786+0000", 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039360456000000002 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467847+0000", 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0847999999999997e-05 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467885+0000", 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8680000000000002e-05 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195370 7.d 7:b9f15733:::benchmark_data_smithi017_220393_object76236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 75.671950409999994, 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14708705, 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195370 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318674+0000", 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.037 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318680+0000", 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0700000000000003e-06 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318691+0000", 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0502e-05 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318695+0000", 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2270000000000004e-06 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318730+0000", 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5132999999999997e-05 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318848+0000", 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118114 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318955+0000", 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000106533 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.321687+0000", 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027322470000000001 2024-09-13T14:39:34.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322829+0000", 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001141853 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465673+0000", 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14284481299999999 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465722+0000", 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8504000000000002e-05 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465762+0000", 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0374999999999999e-05 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.042 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195371 7.7 7:ecd38d35:::benchmark_data_smithi017_220393_object76237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.322857+0000", 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 75.667768280999994, 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14513889299999999, 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195371 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322857+0000", 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322857+0000", 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322856+0000", 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322863+0000", 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6250000000000001e-06 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322876+0000", 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.307e-05 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322883+0000", 2024-09-13T14:39:34.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.306e-06 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322928+0000", 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5117999999999997e-05 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323057+0000", 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012941999999999999 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323175+0000", 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117176 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.329072+0000", 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058975340000000003 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.458525+0000", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12945304599999999 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467926+0000", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0094012120000000008 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467966+0000", 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9521000000000003e-05 2024-09-13T14:39:34.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467996+0000", 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0349e-05 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195372 7.7 7:e306fce3:::benchmark_data_smithi017_220393_object76238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.323000+0000", 2024-09-13T14:39:34.050 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 75.667625409999999, 2024-09-13T14:39:34.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14518144399999999, 2024-09-13T14:39:34.053 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.053 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.053 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.053 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195372 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323000+0000", 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323000+0000", 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323000+0000", 2024-09-13T14:39:34.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323004+0000", 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8459999999999999e-06 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323014+0000", 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4609999999999997e-06 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.323018+0000", 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6720000000000003e-06 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.326311+0000", 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032927099999999999 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.326452+0000", 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014076 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.326585+0000", 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013353200000000001 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.329220+0000", 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026348180000000001 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.458553+0000", 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12933241100000001 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.468038+0000", 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0094849070000000008 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.468139+0000", 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101414 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.468182+0000", 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2676000000000003e-05 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.060 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195907 7.d 7:b93f33ee:::benchmark_data_smithi017_220393_object76773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:19.102485+0000", 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 74.888141020999996, 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.119663273, 2024-09-13T14:39:34.061 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195907 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102485+0000", 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102485+0000", 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102483+0000", 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102494+0000", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0793e-05 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102518+0000", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4482999999999999e-05 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102529+0000", 2024-09-13T14:39:34.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0356e-05 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102595+0000", 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6946000000000006e-05 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102778+0000", 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000182117 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.102878+0000", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9970999999999995e-05 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.105537+0000", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026597589999999998 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.106197+0000", 2024-09-13T14:39:34.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00066008099999999997 2024-09-13T14:39:34.067 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.067 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.068 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.068 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.222041+0000", 2024-09-13T14:39:34.068 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.115843142 2024-09-13T14:39:34.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.068 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.222108+0000", 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7795999999999999e-05 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:19.222148+0000", 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9612e-05 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199290 7.7 7:e5b5f326:::benchmark_data_smithi017_220393_object80156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 70.501378130999996, 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20093702499999999, 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199290 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.070 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489246+0000", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489252+0000", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7209999999999999e-06 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489264+0000", 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2028000000000001e-05 2024-09-13T14:39:34.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489270+0000", 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6219999999999997e-06 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489300+0000", 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0456000000000001e-05 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489418+0000", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117551 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489507+0000", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9654999999999995e-05 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.492597+0000", 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030899809999999999 2024-09-13T14:39:34.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.499941+0000", 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007343848 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690098+0000", 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190156834 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690145+0000", 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7094e-05 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690185+0000", 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9473e-05 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199306 7.d 7:b9985ddb:::benchmark_data_smithi017_220393_object80172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 70.483634007999996, 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.17325623900000001, 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199306 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506991+0000", 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506996+0000", 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0529999999999997e-06 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507007+0000", 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1533e-05 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507013+0000", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8640000000000001e-06 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507057+0000", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4322e-05 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507173+0000", 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011561 2024-09-13T14:39:34.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507291+0000", 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117555 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.510005+0000", 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027142020000000002 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.511252+0000", 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001246788 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680183+0000", 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.16893149399999999 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680220+0000", 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7271999999999998e-05 2024-09-13T14:39:34.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680248+0000", 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7537000000000001e-05 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199321 7.d 7:b50dc68b:::benchmark_data_smithi017_220393_object80187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.564759+0000", 2024-09-13T14:39:34.083 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 70.425867097999998, 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11933392900000001, 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199321 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.084 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564759+0000", 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564759+0000", 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564757+0000", 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564763+0000", 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9660000000000001e-06 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564775+0000", 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1792e-05 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564779+0000", 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2889999999999999e-06 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564851+0000", 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2035999999999999e-05 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.564983+0000", 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013161299999999999 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.565126+0000", 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000142576 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.567790+0000", 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002663997 2024-09-13T14:39:34.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.568672+0000", 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00088228999999999998 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.684004+0000", 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11533252200000001 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.684052+0000", 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7951999999999998e-05 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.684093+0000", 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0169000000000001e-05 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200609 7.d 7:baf945be:::benchmark_data_smithi017_220393_object81475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.563032696999997, 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34117809199999999, 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200609 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427592+0000", 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427598+0000", 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1979999999999998e-06 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427610+0000", 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.185e-05 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427615+0000", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.772e-06 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427649+0000", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3343999999999998e-05 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427739+0000", 2024-09-13T14:39:34.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0749999999999997e-05 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427862+0000", 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012259799999999999 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434044+0000", 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061822839999999997 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435398+0000", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001354035 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768686+0000", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33328797199999999 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768733+0000", 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7101999999999997e-05 2024-09-13T14:39:34.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768771+0000", 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7586000000000002e-05 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200614 7.d 7:bd88b3c8:::benchmark_data_smithi017_220393_object81480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:39:34.097 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.562053632000001, 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34032926800000002, 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200614 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:39:34.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428571+0000", 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428576+0000", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4469999999999999e-06 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428585+0000", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428588+0000", 2024-09-13T14:39:34.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5880000000000002e-06 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.430963+0000", 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023746639999999999 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431063+0000", 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000100412 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431173+0000", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000109085 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434158+0000", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029855020000000001 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435507+0000", 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001348471 2024-09-13T14:39:34.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768816+0000", 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.333309466 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768866+0000", 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9571999999999999e-05 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768901+0000", 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5741000000000002e-05 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200631 7.d 7:b540f86c:::benchmark_data_smithi017_220393_object81497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.423681161000005, 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.202045216, 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.104 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200631 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566943+0000", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566949+0000", 2024-09-13T14:39:34.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4589999999999997e-06 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566960+0000", 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1046999999999999e-05 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566966+0000", 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9630000000000003e-06 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567008+0000", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1878999999999997e-05 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567095+0000", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7145000000000007e-05 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567169+0000", 2024-09-13T14:39:34.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3732000000000005e-05 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.570791+0000", 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036220850000000001 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.571627+0000", 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000835928 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768940+0000", 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19731306200000001 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768970+0000", 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0368000000000001e-05 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768990+0000", 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9626000000000001e-05 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200634 7.d 7:b80f92f6:::benchmark_data_smithi017_220393_object81500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.418031049000007, 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19650853099999999, 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.111 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200634 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572594+0000", 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572600+0000", 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1920000000000002e-06 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572611+0000", 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1569999999999999e-05 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572616+0000", 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2679999999999996e-06 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572656+0000", 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0741000000000002e-05 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572783+0000", 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012668299999999999 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572938+0000", 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000155351 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.575602+0000", 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026634129999999999 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.577024+0000", 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014224999999999999 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769023+0000", 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19199869899999999 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769068+0000", 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5064000000000001e-05 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769103+0000", 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5173999999999999e-05 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200637 7.d 7:b4355551:::benchmark_data_smithi017_220393_object81503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.409015925999995, 2024-09-13T14:39:34.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190526902, 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200637 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581609+0000", 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581614+0000", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9139999999999996e-06 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581626+0000", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1681999999999999e-05 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581632+0000", 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7780000000000004e-06 2024-09-13T14:39:34.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581664+0000", 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1585999999999999e-05 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581742+0000", 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8591999999999996e-05 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581828+0000", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5816000000000006e-05 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584440+0000", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026124199999999998 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584798+0000", 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035729199999999999 2024-09-13T14:39:34.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772042+0000", 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18724463299999999 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772092+0000", 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9638000000000001e-05 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772137+0000", 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4747999999999999e-05 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:202250 7.c 7:3efb95fe:::benchmark_data_smithi017_220393_object83116:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:28.274464+0000", 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 65.716161434, 2024-09-13T14:39:34.125 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.114607027, 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202250 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274464+0000", 2024-09-13T14:39:34.126 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274464+0000", 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274463+0000", 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274469+0000", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9190000000000004e-06 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274482+0000", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.327e-05 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274488+0000", 2024-09-13T14:39:34.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8300000000000001e-06 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274526+0000", 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8427000000000003e-05 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274657+0000", 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000131185 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.274740+0000", 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2665999999999998e-05 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.277922+0000", 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031821010000000001 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.278366+0000", 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00044335700000000002 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.388959+0000", 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11059385100000001 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.389027+0000", 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7887999999999998e-05 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.389071+0000", 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3930999999999997e-05 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:202251 7.7 7:e50579b5:::benchmark_data_smithi017_220393_object83117:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:28.275590+0000", 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 65.715036147000006, 2024-09-13T14:39:34.132 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.113584465, 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202251 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275590+0000", 2024-09-13T14:39:34.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275590+0000", 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275589+0000", 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275594+0000", 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1070000000000004e-06 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275605+0000", 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1453e-05 2024-09-13T14:39:34.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275611+0000", 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9560000000000002e-06 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275652+0000", 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0380999999999998e-05 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275756+0000", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000104209 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.275868+0000", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011262800000000001 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.279258+0000", 2024-09-13T14:39:34.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0033893980000000001 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.284989+0000", 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057309680000000003 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.389020+0000", 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.10403080300000001 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.389139+0000", 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011938300000000001 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.389174+0000", 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5052000000000003e-05 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:202258 7.d 7:bb897a9e:::benchmark_data_smithi017_220393_object83124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:28.280562+0000", 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 65.710063438000006, 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12008489, 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202258 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.140 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280562+0000", 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280562+0000", 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:39:34.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280560+0000", 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280567+0000", 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0480000000000001e-06 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280578+0000", 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0699999999999999e-05 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280584+0000", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0610000000000001e-06 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280629+0000", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4858000000000003e-05 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280750+0000", 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012073100000000001 2024-09-13T14:39:34.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.280883+0000", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000133335 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.283795+0000", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029114010000000001 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.288005+0000", 2024-09-13T14:39:34.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042105670000000001 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.400560+0000", 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.112555368 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.400609+0000", 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8683999999999997e-05 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:28.400647+0000", 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8003999999999997e-05 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.146 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.147 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.147 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:39:34.147 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:39:34.147 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:39:34.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:34.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:34.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:34.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:34.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:39:34.458 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:39:34.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:34.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:34.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[103001]: pgmap v866: 99 pgs: 13 unknown, 22 active+undersized, 9 active+undersized+degraded, 55 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:34.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:34.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[103001]: osdmap e305: 8 total, 7 up, 7 in 2024-09-13T14:39:34.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/487887635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:34.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:34.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[107556]: pgmap v866: 99 pgs: 13 unknown, 22 active+undersized, 9 active+undersized+degraded, 55 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:34.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:34.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[107556]: osdmap e305: 8 total, 7 up, 7 in 2024-09-13T14:39:34.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/487887635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:34.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:34.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:34.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:34 smithi189 ceph-mon[90992]: pgmap v866: 99 pgs: 13 unknown, 22 active+undersized, 9 active+undersized+degraded, 55 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:34 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:39:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:34 smithi189 ceph-mon[90992]: osdmap e305: 8 total, 7 up, 7 in 2024-09-13T14:39:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/487887635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:34.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:34.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:34.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:34.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:35.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:35.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:35.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:35.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:35.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:35.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:35.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:35.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:35.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:39:35.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:35.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:35.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:35.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:35.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:35.772 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:35.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:35.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:35.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:35.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:35.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:35.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:36.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:36.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:36.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:36.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:36.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:36.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:36.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:36.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:36.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:36.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:39:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:36 smithi189 ceph-mon[90992]: pgmap v868: 99 pgs: 4 unknown, 26 active+undersized, 9 active+undersized+degraded, 60 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:36.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.577 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:36.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:36.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:36.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:36 smithi017 ceph-mon[103001]: pgmap v868: 99 pgs: 4 unknown, 26 active+undersized, 9 active+undersized+degraded, 60 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:36.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:36 smithi017 ceph-mon[107556]: pgmap v868: 99 pgs: 4 unknown, 26 active+undersized, 9 active+undersized+degraded, 60 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:36.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:36.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:36.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:36.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:36.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:36.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:37.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:37.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:37.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:37.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:37.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:37.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.444 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:39:37.445 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:39:37.445 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:39:37.445 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:39:37.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:37.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:37.562 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:37.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:37.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:37.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:37.682 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:37] "GET /metrics HTTP/1.1" 200 34315 "" "Prometheus/2.51.0" 2024-09-13T14:39:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.698 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:37.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:37.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:37.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:37.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:38.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:38.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:38.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:39:38.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:38.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:38.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:38.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:38.333 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-13T14:39:38.333 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:39:38.361 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:38.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:38.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:38.482 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:38 smithi189 ceph-mon[90992]: pgmap v869: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:38.483 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/391645105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.483 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:38 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.483 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:38.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:38.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:38.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:38.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[107556]: pgmap v869: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/391645105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[103001]: pgmap v869: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/391645105' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:39:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:38.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:38.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:38.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:38.941 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:38.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:39.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:39.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:39.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:39.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:39.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:39.295 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:39.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:39.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:39.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[107556]: osdmap e306: 8 total, 7 up, 7 in 2024-09-13T14:39:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:39 smithi017 ceph-mon[103001]: osdmap e306: 8 total, 7 up, 7 in 2024-09-13T14:39:39.483 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:39 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:39.483 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:39:39.484 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1871610888' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:39.484 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:39 smithi189 ceph-mon[90992]: osdmap e306: 8 total, 7 up, 7 in 2024-09-13T14:39:39.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:39.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:39.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:39.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:39.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:39.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:39.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:39.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:39.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:39.973 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:39:39.974 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:39:39.974 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:39:39.974 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:39:39.974 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:39:40.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:40.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:40.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:40.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:40.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:40.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:40.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:40.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:40.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:40.496 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-13T14:39:40.497 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:39:40.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:39:40.499 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:39:40.499 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:39:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:40 smithi189 ceph-mon[90992]: pgmap v871: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:40.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:40 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:40.507 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:40.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:40.548 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:39:40.549 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:39:40.549 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_253885 2024-09-13T14:39:40.549 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:39:40.549 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:39:40.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:40.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:40 smithi017 ceph-mon[103001]: pgmap v871: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:40.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:40 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:40.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:40 smithi017 ceph-mon[107556]: pgmap v871: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:40.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:40 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:39:40.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:39:40.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:39:40.656 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:39:40.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:40.698 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:39:40.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:39:40.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:40.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:40.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:39:40.737 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:39:40.775 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:39:40.775 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:39:40.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:39:40.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:39:40.808 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:39:40.823 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:40.843 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:39:40.843 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:39:40.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:40.879 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:39:40.880 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:39:40.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:40.963 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:39:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:40.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:40.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:41.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:41.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:41.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:41.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:41.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:41.437 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1562656986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:41.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:41.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:41.549 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 15 3403 3388 211.738 211.75 0.00369889 0.00471172 2024-09-13T14:39:41.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:41.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1562656986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1562656986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:39:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:39:41.755 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:39:41.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:41.799 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:41.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:41.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:41.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:41.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:42.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:42.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:39:42.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:42.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:42.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:42.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[103001]: pgmap v872: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[107556]: pgmap v872: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:42 smithi189 ceph-mon[90992]: pgmap v872: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 4.5 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 13/84 objects degraded (15.476%) 2024-09-13T14:39:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:39:42.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:42.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:42.550 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6438 6422 200.609 189.625 0.00530283 0.00497309 2024-09-13T14:39:42.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:42.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:42.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:42.821 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:39:42.822 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:39:42.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:42.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:42.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:42.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:42.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:43.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:43.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:43.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:43.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:43.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:43.396 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:43.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:43.453 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 1020/4605 objects degraded (22.150%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:43.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:39:43.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:43.519 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:43.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:43.550 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 15 9538 9523 198.337 193.812 0.00693484 0.00503138 2024-09-13T14:39:43.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:43.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 1020/4605 objects degraded (22.150%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:43.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 1020/4605 objects degraded (22.150%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:43.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:43.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:39:43.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:43.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:43.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:43.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:44.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:44.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:44.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:44.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:44.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:44.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:44.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:44.547 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:44.550 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12079 12063 188.438 158.75 0.00295369 0.00530011 2024-09-13T14:39:44.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:44.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:44 smithi017 ceph-mon[103001]: pgmap v873: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 99 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 9.6 MiB/s wr, 153 op/s; 1020/4605 objects degraded (22.150%) 2024-09-13T14:39:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:44 smithi017 ceph-mon[107556]: pgmap v873: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 99 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 9.6 MiB/s wr, 153 op/s; 1020/4605 objects degraded (22.150%) 2024-09-13T14:39:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:44.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:44 smithi189 ceph-mon[90992]: pgmap v873: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 99 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 9.6 MiB/s wr, 153 op/s; 1020/4605 objects degraded (22.150%) 2024-09-13T14:39:44.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:44.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:44.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:44.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:44.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:44.971 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:39:44.973 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:39:45.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:45.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:45.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:45.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:45.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:45.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:45.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:45.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:45.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:45.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 14741 14725 183.956 166.375 0.0100999 0.00542882 2024-09-13T14:39:45.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:45.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:45.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:45.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:45.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:45.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:45.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:46.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:46.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:46.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:46.205 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:46.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:46.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:46.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:46.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:46.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:46.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 17213 17197 179.047 154.5 0.00570915 0.0055787 2024-09-13T14:39:46.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:46.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:46.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:46 smithi017 ceph-mon[103001]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:46.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:46 smithi017 ceph-mon[103001]: pgmap v874: 99 pgs: 22 active+undersized, 15 active+undersized+degraded, 62 active+clean; 492 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 3792/23463 objects degraded (16.162%) 2024-09-13T14:39:46.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:46 smithi017 ceph-mon[107556]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:46.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:46 smithi017 ceph-mon[107556]: pgmap v874: 99 pgs: 22 active+undersized, 15 active+undersized+degraded, 62 active+clean; 492 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 3792/23463 objects degraded (16.162%) 2024-09-13T14:39:46.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:46 smithi189 ceph-mon[90992]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:39:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:46 smithi189 ceph-mon[90992]: pgmap v874: 99 pgs: 22 active+undersized, 15 active+undersized+degraded, 62 active+clean; 492 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 3792/23463 objects degraded (16.162%) 2024-09-13T14:39:46.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:46.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:46.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:46.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:47.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:47.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:47.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:47.155 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":873,"stamp":"2024-09-13T14:39:43.110388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":103504964,"num_objects":1535,"num_object_clones":0,"num_object_copies":4605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1020,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1535,"num_whiteouts":0,"num_read":6074,"num_read_kb":13253,"num_write":2468,"num_write_kb":109663,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":219569,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3639,"ondisk_log_size":3639,"up":260,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":294,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3920484,"kb_used_data":558164,"kb_used_omap":34,"kb_used_meta":3362141,"kb_avail":652295580,"statfs":{"total":671965249536,"available":667950673920,"internally_reserved":0,"allocated":571559936,"data_stored":561111838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":3442832431},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1007,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1507,"num_write_kb":96448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.792277"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":493,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546805+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:39:38.546805+0000","last_peered":"2024-09-13T14:39:38.546805+0000","last_clean":"2024-09-13T14:39:38.546805+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:39:38.546805+0000","last_undegraded":"2024-09-13T14:39:38.546805+0000","last_fullsized":"2024-09-13T14:39:38.546805+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":287,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.313031+0000","last_change":"2024-09-13T14:38:03.566972+0000","last_active":"2024-09-13T14:39:32.313031+0000","last_peered":"2024-09-13T14:39:32.313031+0000","last_clean":"2024-09-13T14:39:32.313031+0000","last_became_active":"2024-09-13T14:38:03.561212+0000","last_became_peered":"2024-09-13T14:38:03.561212+0000","last_unstale":"2024-09-13T14:39:32.313031+0000","last_undegraded":"2024-09-13T14:39:32.313031+0000","last_fullsized":"2024-09-13T14:39:32.313031+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:41.501833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":304,"state":"active+undersized","last_fresh":"2024-09-13T14:39:32.313040+0000","last_change":"2024-09-13T14:39:00.143278+0000","last_active":"2024-09-13T14:39:32.313040+0000","last_peered":"2024-09-13T14:39:32.313040+0000","last_clean":"2024-09-13T14:23:27.712756+0000","last_became_active":"2024-09-13T14:39:00.143278+0000","last_became_peered":"2024-09-13T14:39:00.143278+0000","last_unstale":"2024-09-13T14:39:32.313040+0000","last_undegraded":"2024-09-13T14:39:32.313040+0000","last_fullsized":"2024-09-13T14:39:00.141577+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:35.530384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":478,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331243+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:39:38.331243+0000","last_peered":"2024-09-13T14:39:38.331243+0000","last_clean":"2024-09-13T14:39:38.331243+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:39:38.331243+0000","last_undegraded":"2024-09-13T14:39:38.331243+0000","last_fullsized":"2024-09-13T14:39:38.331243+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":479,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331924+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:39:38.331924+0000","last_peered":"2024-09-13T14:39:38.331924+0000","last_clean":"2024-09-13T14:39:38.331924+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:39:38.331924+0000","last_undegraded":"2024-09-13T14:39:38.331924+0000","last_fullsized":"2024-09-13T14:39:38.331924+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":262,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312628+0000","last_change":"2024-09-13T14:38:03.566493+0000","last_active":"2024-09-13T14:39:32.312628+0000","last_peered":"2024-09-13T14:39:32.312628+0000","last_clean":"2024-09-13T14:39:32.312628+0000","last_became_active":"2024-09-13T14:38:03.566174+0000","last_became_peered":"2024-09-13T14:38:03.566174+0000","last_unstale":"2024-09-13T14:39:32.312628+0000","last_undegraded":"2024-09-13T14:39:32.312628+0000","last_fullsized":"2024-09-13T14:39:32.312628+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.777867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":493,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.547191+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:39:38.547191+0000","last_peered":"2024-09-13T14:39:38.547191+0000","last_clean":"2024-09-13T14:39:38.547191+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:39:38.547191+0000","last_undegraded":"2024-09-13T14:39:38.547191+0000","last_fullsized":"2024-09-13T14:39:38.547191+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":285,"reported_epoch":305,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:33.319685+0000","last_change":"2024-09-13T14:39:00.146202+0000","last_active":"2024-09-13T14:39:33.319685+0000","last_peered":"2024-09-13T14:39:33.319685+0000","last_clean":"2024-09-13T14:38:58.141715+0000","last_became_active":"2024-09-13T14:39:00.146202+0000","last_became_peered":"2024-09-13T14:39:00.146202+0000","last_unstale":"2024-09-13T14:39:33.319685+0000","last_undegraded":"2024-09-13T14:39:00.138441+0000","last_fullsized":"2024-09-13T14:39:00.138317+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:30:30.676417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"306'489","reported_seq":497,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:43.020437+0000","last_change":"2024-09-13T14:39:40.570618+0000","last_active":"2024-09-13T14:39:43.020437+0000","last_peered":"2024-09-13T14:39:43.020437+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.489850+0000","last_became_peered":"2024-09-13T14:39:32.489850+0000","last_unstale":"2024-09-13T14:39:43.020437+0000","last_undegraded":"2024-09-13T14:39:40.568046+0000","last_fullsized":"2024-09-13T14:39:32.309355+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":489,"log_dups_size":0,"ondisk_log_size":489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:50.934831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32047104,"num_objects":489,"num_object_clones":0,"num_object_copies":1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":489,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":489,"num_write_kb":31296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":489}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":284,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312309+0000","last_change":"2024-09-13T14:38:03.617493+0000","last_active":"2024-09-13T14:39:32.312309+0000","last_peered":"2024-09-13T14:39:32.312309+0000","last_clean":"2024-09-13T14:39:32.312309+0000","last_became_active":"2024-09-13T14:38:03.569854+0000","last_became_peered":"2024-09-13T14:38:03.569854+0000","last_unstale":"2024-09-13T14:39:32.312309+0000","last_undegraded":"2024-09-13T14:39:32.312309+0000","last_fullsized":"2024-09-13T14:39:32.312309+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:17.890012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":20,"reported_epoch":304,"state":"active+undersized","last_fresh":"2024-09-13T14:39:32.311878+0000","last_change":"2024-09-13T14:39:00.143910+0000","last_active":"2024-09-13T14:39:32.311878+0000","last_peered":"2024-09-13T14:39:32.311878+0000","last_clean":"2024-09-13T14:13:41.111573+0000","last_became_active":"2024-09-13T14:39:00.143910+0000","last_became_peered":"2024-09-13T14:39:00.143910+0000","last_unstale":"2024-09-13T14:39:32.311878+0000","last_undegraded":"2024-09-13T14:39:32.311878+0000","last_fullsized":"2024-09-13T14:39:00.140726+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:03:34.681406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"306'500","reported_seq":511,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:43.020211+0000","last_change":"2024-09-13T14:39:32.315906+0000","last_active":"2024-09-13T14:39:43.020211+0000","last_peered":"2024-09-13T14:39:43.020211+0000","last_clean":"2024-09-13T14:39:43.020211+0000","last_became_active":"2024-09-13T14:39:32.315587+0000","last_became_peered":"2024-09-13T14:39:32.315587+0000","last_unstale":"2024-09-13T14:39:43.020211+0000","last_undegraded":"2024-09-13T14:39:43.020211+0000","last_fullsized":"2024-09-13T14:39:43.020211+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":500,"log_dups_size":0,"ondisk_log_size":500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:34.194710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":500,"num_write_kb":32000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":302,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312455+0000","last_change":"2024-09-13T14:38:03.622235+0000","last_active":"2024-09-13T14:39:32.312455+0000","last_peered":"2024-09-13T14:39:32.312455+0000","last_clean":"2024-09-13T14:39:32.312455+0000","last_became_active":"2024-09-13T14:38:03.570116+0000","last_became_peered":"2024-09-13T14:38:03.570116+0000","last_unstale":"2024-09-13T14:39:32.312455+0000","last_undegraded":"2024-09-13T14:39:32.312455+0000","last_fullsized":"2024-09-13T14:39:32.312455+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:24.128120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":313,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545258+0000","last_change":"2024-09-13T14:38:03.571764+0000","last_active":"2024-09-13T14:39:38.545258+0000","last_peered":"2024-09-13T14:39:38.545258+0000","last_clean":"2024-09-13T14:39:38.545258+0000","last_became_active":"2024-09-13T14:38:03.571230+0000","last_became_peered":"2024-09-13T14:38:03.571230+0000","last_unstale":"2024-09-13T14:39:38.545258+0000","last_undegraded":"2024-09-13T14:39:38.545258+0000","last_fullsized":"2024-09-13T14:39:38.545258+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:26:17.921751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331775+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:39:38.331775+0000","last_peered":"2024-09-13T14:39:38.331775+0000","last_clean":"2024-09-13T14:39:38.331775+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:39:38.331775+0000","last_undegraded":"2024-09-13T14:39:38.331775+0000","last_fullsized":"2024-09-13T14:39:38.331775+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":260,"reported_epoch":304,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:32.313253+0000","last_change":"2024-09-13T14:39:00.143754+0000","last_active":"2024-09-13T14:39:32.313253+0000","last_peered":"2024-09-13T14:39:32.313253+0000","last_clean":"2024-09-13T14:38:58.140416+0000","last_became_active":"2024-09-13T14:39:00.143754+0000","last_became_peered":"2024-09-13T14:39:00.143754+0000","last_unstale":"2024-09-13T14:39:32.313253+0000","last_undegraded":"2024-09-13T14:39:00.140881+0000","last_fullsized":"2024-09-13T14:39:00.140747+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:39.745825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":286,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.319906+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:39:33.319906+0000","last_peered":"2024-09-13T14:39:33.319906+0000","last_clean":"2024-09-13T14:39:33.319906+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:39:33.319906+0000","last_undegraded":"2024-09-13T14:39:33.319906+0000","last_fullsized":"2024-09-13T14:39:33.319906+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545906+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:39:38.545906+0000","last_peered":"2024-09-13T14:39:38.545906+0000","last_clean":"2024-09-13T14:39:38.545906+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:39:38.545906+0000","last_undegraded":"2024-09-13T14:39:38.545906+0000","last_fullsized":"2024-09-13T14:39:38.545906+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.546966+0000","last_change":"2024-09-13T14:39:00.150112+0000","last_active":"2024-09-13T14:39:38.546966+0000","last_peered":"2024-09-13T14:39:38.546966+0000","last_clean":"2024-09-13T14:23:27.712618+0000","last_became_active":"2024-09-13T14:39:00.150112+0000","last_became_peered":"2024-09-13T14:39:00.150112+0000","last_unstale":"2024-09-13T14:39:38.546966+0000","last_undegraded":"2024-09-13T14:39:38.546966+0000","last_fullsized":"2024-09-13T14:39:00.138863+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:08:23.729274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":498,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331439+0000","last_change":"2024-09-13T14:38:03.576970+0000","last_active":"2024-09-13T14:39:38.331439+0000","last_peered":"2024-09-13T14:39:38.331439+0000","last_clean":"2024-09-13T14:39:38.331439+0000","last_became_active":"2024-09-13T14:38:03.576495+0000","last_became_peered":"2024-09-13T14:38:03.576495+0000","last_unstale":"2024-09-13T14:39:38.331439+0000","last_undegraded":"2024-09-13T14:39:38.331439+0000","last_fullsized":"2024-09-13T14:39:38.331439+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:30:52.911056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":14,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545088+0000","last_change":"2024-09-13T14:39:32.319970+0000","last_active":"2024-09-13T14:39:38.545088+0000","last_peered":"2024-09-13T14:39:38.545088+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.319970+0000","last_became_peered":"2024-09-13T14:39:32.319970+0000","last_unstale":"2024-09-13T14:39:38.545088+0000","last_undegraded":"2024-09-13T14:39:38.545088+0000","last_fullsized":"2024-09-13T14:39:32.311561+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:56.173485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":416,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312547+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:39:32.312547+0000","last_peered":"2024-09-13T14:39:32.312547+0000","last_clean":"2024-09-13T14:39:32.312547+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:39:32.312547+0000","last_undegraded":"2024-09-13T14:39:32.312547+0000","last_fullsized":"2024-09-13T14:39:32.312547+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":491,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546376+0000","last_change":"2024-09-13T14:38:03.614233+0000","last_active":"2024-09-13T14:39:38.546376+0000","last_peered":"2024-09-13T14:39:38.546376+0000","last_clean":"2024-09-13T14:39:38.546376+0000","last_became_active":"2024-09-13T14:38:03.578168+0000","last_became_peered":"2024-09-13T14:38:03.578168+0000","last_unstale":"2024-09-13T14:39:38.546376+0000","last_undegraded":"2024-09-13T14:39:38.546376+0000","last_fullsized":"2024-09-13T14:39:38.546376+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:59.664708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546998+0000","last_change":"2024-09-13T14:38:03.568127+0000","last_active":"2024-09-13T14:39:38.546998+0000","last_peered":"2024-09-13T14:39:38.546998+0000","last_clean":"2024-09-13T14:39:38.546998+0000","last_became_active":"2024-09-13T14:38:03.567585+0000","last_became_peered":"2024-09-13T14:38:03.567585+0000","last_unstale":"2024-09-13T14:39:38.546998+0000","last_undegraded":"2024-09-13T14:39:38.546998+0000","last_fullsized":"2024-09-13T14:39:38.546998+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:42.094924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545035+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:39:38.545035+0000","last_peered":"2024-09-13T14:39:38.545035+0000","last_clean":"2024-09-13T14:39:38.545035+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:39:38.545035+0000","last_undegraded":"2024-09-13T14:39:38.545035+0000","last_fullsized":"2024-09-13T14:39:38.545035+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":23,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545324+0000","last_change":"2024-09-13T14:39:23.286977+0000","last_active":"2024-09-13T14:39:38.545324+0000","last_peered":"2024-09-13T14:39:38.545324+0000","last_clean":"2024-09-13T14:18:14.078049+0000","last_became_active":"2024-09-13T14:39:23.286977+0000","last_became_peered":"2024-09-13T14:39:23.286977+0000","last_unstale":"2024-09-13T14:39:38.545324+0000","last_undegraded":"2024-09-13T14:39:38.545324+0000","last_fullsized":"2024-09-13T14:39:23.278462+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T14:39:39.622826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":297,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.319760+0000","last_change":"2024-09-13T14:38:03.571256+0000","last_active":"2024-09-13T14:39:33.319760+0000","last_peered":"2024-09-13T14:39:33.319760+0000","last_clean":"2024-09-13T14:39:33.319760+0000","last_became_active":"2024-09-13T14:38:03.570928+0000","last_became_peered":"2024-09-13T14:38:03.570928+0000","last_unstale":"2024-09-13T14:39:33.319760+0000","last_undegraded":"2024-09-13T14:39:33.319760+0000","last_fullsized":"2024-09-13T14:39:33.319760+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:42.569492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":134,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.320259+0000","last_change":"2024-09-13T14:39:23.288895+0000","last_active":"2024-09-13T14:39:33.320259+0000","last_peered":"2024-09-13T14:39:33.320259+0000","last_clean":"2024-09-13T14:38:58.138548+0000","last_became_active":"2024-09-13T14:39:23.288895+0000","last_became_peered":"2024-09-13T14:39:23.288895+0000","last_unstale":"2024-09-13T14:39:33.320259+0000","last_undegraded":"2024-09-13T14:39:33.320259+0000","last_fullsized":"2024-09-13T14:39:23.285623+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:17.072411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":14,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.331070+0000","last_change":"2024-09-13T14:39:32.315624+0000","last_active":"2024-09-13T14:39:38.331070+0000","last_peered":"2024-09-13T14:39:38.331070+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.315624+0000","last_became_peered":"2024-09-13T14:39:32.315624+0000","last_unstale":"2024-09-13T14:39:38.331070+0000","last_undegraded":"2024-09-13T14:39:38.331070+0000","last_fullsized":"2024-09-13T14:39:32.311621+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:01:14.476549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312976+0000","last_change":"2024-09-13T14:39:23.285947+0000","last_active":"2024-09-13T14:39:32.312976+0000","last_peered":"2024-09-13T14:39:32.312976+0000","last_clean":"2024-09-13T14:39:32.312976+0000","last_became_active":"2024-09-13T14:39:23.285671+0000","last_became_peered":"2024-09-13T14:39:23.285671+0000","last_unstale":"2024-09-13T14:39:32.312976+0000","last_undegraded":"2024-09-13T14:39:32.312976+0000","last_fullsized":"2024-09-13T14:39:32.312976+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:56:31.550107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":263,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.320028+0000","last_change":"2024-09-13T14:38:03.573525+0000","last_active":"2024-09-13T14:39:33.320028+0000","last_peered":"2024-09-13T14:39:33.320028+0000","last_clean":"2024-09-13T14:39:33.320028+0000","last_became_active":"2024-09-13T14:38:03.572863+0000","last_became_peered":"2024-09-13T14:38:03.572863+0000","last_unstale":"2024-09-13T14:39:33.320028+0000","last_undegraded":"2024-09-13T14:39:33.320028+0000","last_fullsized":"2024-09-13T14:39:33.320028+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:04:10.340538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":14,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545700+0000","last_change":"2024-09-13T14:39:32.319871+0000","last_active":"2024-09-13T14:39:38.545700+0000","last_peered":"2024-09-13T14:39:38.545700+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.319871+0000","last_became_peered":"2024-09-13T14:39:32.319871+0000","last_unstale":"2024-09-13T14:39:38.545700+0000","last_undegraded":"2024-09-13T14:39:38.545700+0000","last_fullsized":"2024-09-13T14:39:32.316801+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:08.372037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":161,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.319620+0000","last_change":"2024-09-13T14:39:23.288541+0000","last_active":"2024-09-13T14:39:33.319620+0000","last_peered":"2024-09-13T14:39:33.319620+0000","last_clean":"2024-09-13T14:18:14.078049+0000","last_became_active":"2024-09-13T14:39:23.288541+0000","last_became_peered":"2024-09-13T14:39:23.288541+0000","last_unstale":"2024-09-13T14:39:33.319620+0000","last_undegraded":"2024-09-13T14:39:33.319620+0000","last_fullsized":"2024-09-13T14:39:23.285568+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":495,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545630+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:39:38.545630+0000","last_peered":"2024-09-13T14:39:38.545630+0000","last_clean":"2024-09-13T14:39:38.545630+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:39:38.545630+0000","last_undegraded":"2024-09-13T14:39:38.545630+0000","last_fullsized":"2024-09-13T14:39:38.545630+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":450,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546283+0000","last_change":"2024-09-13T14:38:03.563815+0000","last_active":"2024-09-13T14:39:38.546283+0000","last_peered":"2024-09-13T14:39:38.546283+0000","last_clean":"2024-09-13T14:39:38.546283+0000","last_became_active":"2024-09-13T14:38:03.563464+0000","last_became_peered":"2024-09-13T14:38:03.563464+0000","last_unstale":"2024-09-13T14:39:38.546283+0000","last_undegraded":"2024-09-13T14:39:38.546283+0000","last_fullsized":"2024-09-13T14:39:38.546283+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:01.276859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"306'518","reported_seq":526,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:43.023218+0000","last_change":"2024-09-13T14:39:40.566438+0000","last_active":"2024-09-13T14:39:43.023218+0000","last_peered":"2024-09-13T14:39:43.023218+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.316763+0000","last_became_peered":"2024-09-13T14:39:32.316763+0000","last_unstale":"2024-09-13T14:39:43.023218+0000","last_undegraded":"2024-09-13T14:39:40.559234+0000","last_fullsized":"2024-09-13T14:39:32.309355+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":518,"log_dups_size":0,"ondisk_log_size":518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:28:55.997983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33947648,"num_objects":518,"num_object_clones":0,"num_object_copies":1554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":518,"num_write_kb":33152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":518}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":407,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312201+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:39:32.312201+0000","last_peered":"2024-09-13T14:39:32.312201+0000","last_clean":"2024-09-13T14:39:32.312201+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:39:32.312201+0000","last_undegraded":"2024-09-13T14:39:32.312201+0000","last_fullsized":"2024-09-13T14:39:32.312201+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":638,"reported_epoch":304,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:32.312831+0000","last_change":"2024-09-13T14:39:23.286454+0000","last_active":"2024-09-13T14:39:32.312831+0000","last_peered":"2024-09-13T14:39:32.312831+0000","last_clean":"2024-09-13T14:15:16.069569+0000","last_became_active":"2024-09-13T14:39:23.286454+0000","last_became_peered":"2024-09-13T14:39:23.286454+0000","last_unstale":"2024-09-13T14:39:32.312831+0000","last_undegraded":"2024-09-13T14:39:23.280156+0000","last_fullsized":"2024-09-13T14:39:23.279980+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:05.213757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":441,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545587+0000","last_change":"2024-09-13T14:39:00.144641+0000","last_active":"2024-09-13T14:39:38.545587+0000","last_peered":"2024-09-13T14:39:38.545587+0000","last_clean":"2024-09-13T14:38:58.152731+0000","last_became_active":"2024-09-13T14:39:00.144641+0000","last_became_peered":"2024-09-13T14:39:00.144641+0000","last_unstale":"2024-09-13T14:39:38.545587+0000","last_undegraded":"2024-09-13T14:39:38.545587+0000","last_fullsized":"2024-09-13T14:39:00.138082+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:58:04.114501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":262,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.320458+0000","last_change":"2024-09-13T14:38:03.568729+0000","last_active":"2024-09-13T14:39:33.320458+0000","last_peered":"2024-09-13T14:39:33.320458+0000","last_clean":"2024-09-13T14:39:33.320458+0000","last_became_active":"2024-09-13T14:38:03.568266+0000","last_became_peered":"2024-09-13T14:38:03.568266+0000","last_unstale":"2024-09-13T14:39:33.320458+0000","last_undegraded":"2024-09-13T14:39:33.320458+0000","last_fullsized":"2024-09-13T14:39:33.320458+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:50.779935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":14,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545245+0000","last_change":"2024-09-13T14:39:32.491019+0000","last_active":"2024-09-13T14:39:38.545245+0000","last_peered":"2024-09-13T14:39:38.545245+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.491019+0000","last_became_peered":"2024-09-13T14:39:32.491019+0000","last_unstale":"2024-09-13T14:39:38.545245+0000","last_undegraded":"2024-09-13T14:39:38.545245+0000","last_fullsized":"2024-09-13T14:39:32.318478+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:30.624146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":294,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.546186+0000","last_change":"2024-09-13T14:39:00.150189+0000","last_active":"2024-09-13T14:39:38.546186+0000","last_peered":"2024-09-13T14:39:38.546186+0000","last_clean":"2024-09-13T14:38:58.150210+0000","last_became_active":"2024-09-13T14:39:00.150189+0000","last_became_peered":"2024-09-13T14:39:00.150189+0000","last_unstale":"2024-09-13T14:39:38.546186+0000","last_undegraded":"2024-09-13T14:39:38.546186+0000","last_fullsized":"2024-09-13T14:39:00.139837+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:56.163999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":695,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312120+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:39:32.312120+0000","last_peered":"2024-09-13T14:39:32.312120+0000","last_clean":"2024-09-13T14:39:32.312120+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:39:32.312120+0000","last_undegraded":"2024-09-13T14:39:32.312120+0000","last_fullsized":"2024-09-13T14:39:32.312120+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":490,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.331342+0000","last_change":"2024-09-13T14:39:00.153992+0000","last_active":"2024-09-13T14:39:38.331342+0000","last_peered":"2024-09-13T14:39:38.331342+0000","last_clean":"2024-09-13T14:38:58.145748+0000","last_became_active":"2024-09-13T14:39:00.153992+0000","last_became_peered":"2024-09-13T14:39:00.153992+0000","last_unstale":"2024-09-13T14:39:38.331342+0000","last_undegraded":"2024-09-13T14:39:38.331342+0000","last_fullsized":"2024-09-13T14:39:00.144227+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:09.694510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":502,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.332123+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:39:38.332123+0000","last_peered":"2024-09-13T14:39:38.332123+0000","last_clean":"2024-09-13T14:39:38.332123+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:39:38.332123+0000","last_undegraded":"2024-09-13T14:39:38.332123+0000","last_fullsized":"2024-09-13T14:39:38.332123+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546065+0000","last_change":"2024-09-13T14:39:32.491002+0000","last_active":"2024-09-13T14:39:38.546065+0000","last_peered":"2024-09-13T14:39:38.546065+0000","last_clean":"2024-09-13T14:39:38.546065+0000","last_became_active":"2024-09-13T14:39:32.490691+0000","last_became_peered":"2024-09-13T14:39:32.490691+0000","last_unstale":"2024-09-13T14:39:38.546065+0000","last_undegraded":"2024-09-13T14:39:38.546065+0000","last_fullsized":"2024-09-13T14:39:38.546065+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:27.780258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":426,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312037+0000","last_change":"2024-09-13T14:38:03.566312+0000","last_active":"2024-09-13T14:39:32.312037+0000","last_peered":"2024-09-13T14:39:32.312037+0000","last_clean":"2024-09-13T14:39:32.312037+0000","last_became_active":"2024-09-13T14:38:03.565875+0000","last_became_peered":"2024-09-13T14:38:03.565875+0000","last_unstale":"2024-09-13T14:39:32.312037+0000","last_undegraded":"2024-09-13T14:39:32.312037+0000","last_fullsized":"2024-09-13T14:39:32.312037+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:05.346013+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":466,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545309+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:39:38.545309+0000","last_peered":"2024-09-13T14:39:38.545309+0000","last_clean":"2024-09-13T14:39:38.545309+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:39:38.545309+0000","last_undegraded":"2024-09-13T14:39:38.545309+0000","last_fullsized":"2024-09-13T14:39:38.545309+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":434,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.544917+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:39:38.544917+0000","last_peered":"2024-09-13T14:39:38.544917+0000","last_clean":"2024-09-13T14:39:38.544917+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:39:38.544917+0000","last_undegraded":"2024-09-13T14:39:38.544917+0000","last_fullsized":"2024-09-13T14:39:38.544917+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":484,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.332211+0000","last_change":"2024-09-13T14:39:23.285771+0000","last_active":"2024-09-13T14:39:38.332211+0000","last_peered":"2024-09-13T14:39:38.332211+0000","last_clean":"2024-09-13T14:38:58.139108+0000","last_became_active":"2024-09-13T14:39:23.285771+0000","last_became_peered":"2024-09-13T14:39:23.285771+0000","last_unstale":"2024-09-13T14:39:38.332211+0000","last_undegraded":"2024-09-13T14:39:38.332211+0000","last_fullsized":"2024-09-13T14:39:23.283135+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:40.376905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":14,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545037+0000","last_change":"2024-09-13T14:39:32.317457+0000","last_active":"2024-09-13T14:39:38.545037+0000","last_peered":"2024-09-13T14:39:38.545037+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.317457+0000","last_became_peered":"2024-09-13T14:39:32.317457+0000","last_unstale":"2024-09-13T14:39:38.545037+0000","last_undegraded":"2024-09-13T14:39:38.545037+0000","last_fullsized":"2024-09-13T14:39:32.312323+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:05:29.990513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":175,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:38.331122+0000","last_change":"2024-09-13T14:39:00.154350+0000","last_active":"2024-09-13T14:39:38.331122+0000","last_peered":"2024-09-13T14:39:38.331122+0000","last_clean":"2024-09-13T14:18:14.078049+0000","last_became_active":"2024-09-13T14:39:00.154350+0000","last_became_peered":"2024-09-13T14:39:00.154350+0000","last_unstale":"2024-09-13T14:39:38.331122+0000","last_undegraded":"2024-09-13T14:39:00.145177+0000","last_fullsized":"2024-09-13T14:39:00.145047+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T14:39:38.396600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":673,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:38.545466+0000","last_change":"2024-09-13T14:39:00.145937+0000","last_active":"2024-09-13T14:39:38.545466+0000","last_peered":"2024-09-13T14:39:38.545466+0000","last_clean":"2024-09-13T14:15:15.054664+0000","last_became_active":"2024-09-13T14:39:00.145937+0000","last_became_peered":"2024-09-13T14:39:00.145937+0000","last_unstale":"2024-09-13T14:39:38.545466+0000","last_undegraded":"2024-09-13T14:39:00.139391+0000","last_fullsized":"2024-09-13T14:39:00.139280+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:56.954327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331106+0000","last_change":"2024-09-13T14:38:03.574313+0000","last_active":"2024-09-13T14:39:38.331106+0000","last_peered":"2024-09-13T14:39:38.331106+0000","last_clean":"2024-09-13T14:39:38.331106+0000","last_became_active":"2024-09-13T14:38:03.573500+0000","last_became_peered":"2024-09-13T14:38:03.573500+0000","last_unstale":"2024-09-13T14:39:38.331106+0000","last_undegraded":"2024-09-13T14:39:38.331106+0000","last_fullsized":"2024-09-13T14:39:38.331106+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:31.718072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":455,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546584+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:39:38.546584+0000","last_peered":"2024-09-13T14:39:38.546584+0000","last_clean":"2024-09-13T14:39:38.546584+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:39:38.546584+0000","last_undegraded":"2024-09-13T14:39:38.546584+0000","last_fullsized":"2024-09-13T14:39:38.546584+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331570+0000","last_change":"2024-09-13T14:39:32.489409+0000","last_active":"2024-09-13T14:39:38.331570+0000","last_peered":"2024-09-13T14:39:38.331570+0000","last_clean":"2024-09-13T14:39:38.331570+0000","last_became_active":"2024-09-13T14:39:32.489093+0000","last_became_peered":"2024-09-13T14:39:32.489093+0000","last_unstale":"2024-09-13T14:39:38.331570+0000","last_undegraded":"2024-09-13T14:39:38.331570+0000","last_fullsized":"2024-09-13T14:39:38.331570+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:47.851037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":305,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.320075+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:39:33.320075+0000","last_peered":"2024-09-13T14:39:33.320075+0000","last_clean":"2024-09-13T14:39:33.320075+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:39:33.320075+0000","last_undegraded":"2024-09-13T14:39:33.320075+0000","last_fullsized":"2024-09-13T14:39:33.320075+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":104,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.547116+0000","last_change":"2024-09-13T14:39:00.153936+0000","last_active":"2024-09-13T14:39:38.547116+0000","last_peered":"2024-09-13T14:39:38.547116+0000","last_clean":"2024-09-13T14:23:27.723029+0000","last_became_active":"2024-09-13T14:39:00.153936+0000","last_became_peered":"2024-09-13T14:39:00.153936+0000","last_unstale":"2024-09-13T14:39:38.547116+0000","last_undegraded":"2024-09-13T14:39:38.547116+0000","last_fullsized":"2024-09-13T14:39:00.138169+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:42:15.206694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1034,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546815+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:39:38.546815+0000","last_peered":"2024-09-13T14:39:38.546815+0000","last_clean":"2024-09-13T14:39:38.546815+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:39:38.546815+0000","last_undegraded":"2024-09-13T14:39:38.546815+0000","last_fullsized":"2024-09-13T14:39:38.546815+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":424,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.545831+0000","last_change":"2024-09-13T14:39:00.146967+0000","last_active":"2024-09-13T14:39:38.545831+0000","last_peered":"2024-09-13T14:39:38.545831+0000","last_clean":"2024-09-13T14:38:58.147705+0000","last_became_active":"2024-09-13T14:39:00.146967+0000","last_became_peered":"2024-09-13T14:39:00.146967+0000","last_unstale":"2024-09-13T14:39:38.545831+0000","last_undegraded":"2024-09-13T14:39:38.545831+0000","last_fullsized":"2024-09-13T14:39:00.140547+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:28.043346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.547309+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:39:38.547309+0000","last_peered":"2024-09-13T14:39:38.547309+0000","last_clean":"2024-09-13T14:39:38.547309+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:39:38.547309+0000","last_undegraded":"2024-09-13T14:39:38.547309+0000","last_fullsized":"2024-09-13T14:39:38.547309+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":305,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.319924+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:39:33.319924+0000","last_peered":"2024-09-13T14:39:33.319924+0000","last_clean":"2024-09-13T14:39:33.319924+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:39:33.319924+0000","last_undegraded":"2024-09-13T14:39:33.319924+0000","last_fullsized":"2024-09-13T14:39:33.319924+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":438,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.546672+0000","last_change":"2024-09-13T14:39:00.150415+0000","last_active":"2024-09-13T14:39:38.546672+0000","last_peered":"2024-09-13T14:39:38.546672+0000","last_clean":"2024-09-13T14:38:58.147621+0000","last_became_active":"2024-09-13T14:39:00.150415+0000","last_became_peered":"2024-09-13T14:39:00.150415+0000","last_unstale":"2024-09-13T14:39:38.546672+0000","last_undegraded":"2024-09-13T14:39:38.546672+0000","last_fullsized":"2024-09-13T14:39:00.140736+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:07.076289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":657,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.313424+0000","last_change":"2024-09-13T14:38:06.593527+0000","last_active":"2024-09-13T14:39:32.313424+0000","last_peered":"2024-09-13T14:39:32.313424+0000","last_clean":"2024-09-13T14:39:32.313424+0000","last_became_active":"2024-09-13T14:38:06.593161+0000","last_became_peered":"2024-09-13T14:38:06.593161+0000","last_unstale":"2024-09-13T14:39:32.313424+0000","last_undegraded":"2024-09-13T14:39:32.313424+0000","last_fullsized":"2024-09-13T14:39:32.313424+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:21.866277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":473,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.331235+0000","last_change":"2024-09-13T14:39:00.148826+0000","last_active":"2024-09-13T14:39:38.331235+0000","last_peered":"2024-09-13T14:39:38.331235+0000","last_clean":"2024-09-13T14:38:58.144557+0000","last_became_active":"2024-09-13T14:39:00.148826+0000","last_became_peered":"2024-09-13T14:39:00.148826+0000","last_unstale":"2024-09-13T14:39:38.331235+0000","last_undegraded":"2024-09-13T14:39:38.331235+0000","last_fullsized":"2024-09-13T14:39:00.145795+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:57.593936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":17,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331415+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:39:38.331415+0000","last_peered":"2024-09-13T14:39:38.331415+0000","last_clean":"2024-09-13T14:39:38.331415+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:39:38.331415+0000","last_undegraded":"2024-09-13T14:39:38.331415+0000","last_fullsized":"2024-09-13T14:39:38.331415+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":161,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.320270+0000","last_change":"2024-09-13T14:39:23.288552+0000","last_active":"2024-09-13T14:39:33.320270+0000","last_peered":"2024-09-13T14:39:33.320270+0000","last_clean":"2024-09-13T14:18:14.078049+0000","last_became_active":"2024-09-13T14:39:23.288552+0000","last_became_peered":"2024-09-13T14:39:23.288552+0000","last_unstale":"2024-09-13T14:39:33.320270+0000","last_undegraded":"2024-09-13T14:39:33.320270+0000","last_fullsized":"2024-09-13T14:39:23.286145+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":481,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.547358+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:39:38.547358+0000","last_peered":"2024-09-13T14:39:38.547358+0000","last_clean":"2024-09-13T14:39:38.547358+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:39:38.547358+0000","last_undegraded":"2024-09-13T14:39:38.547358+0000","last_fullsized":"2024-09-13T14:39:38.547358+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":793,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546861+0000","last_change":"2024-09-13T14:38:03.590316+0000","last_active":"2024-09-13T14:39:38.546861+0000","last_peered":"2024-09-13T14:39:38.546861+0000","last_clean":"2024-09-13T14:39:38.546861+0000","last_became_active":"2024-09-13T14:38:03.572606+0000","last_became_peered":"2024-09-13T14:38:03.572606+0000","last_unstale":"2024-09-13T14:39:38.546861+0000","last_undegraded":"2024-09-13T14:39:38.546861+0000","last_fullsized":"2024-09-13T14:39:38.546861+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:08.052165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":478,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.547172+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:39:38.547172+0000","last_peered":"2024-09-13T14:39:38.547172+0000","last_clean":"2024-09-13T14:39:38.547172+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:39:38.547172+0000","last_undegraded":"2024-09-13T14:39:38.547172+0000","last_fullsized":"2024-09-13T14:39:38.547172+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":7,"reported_epoch":304,"state":"active+undersized","last_fresh":"2024-09-13T14:39:32.489229+0000","last_change":"2024-09-13T14:39:32.489072+0000","last_active":"2024-09-13T14:39:32.489229+0000","last_peered":"2024-09-13T14:39:32.489229+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:39:32.489072+0000","last_became_peered":"2024-09-13T14:39:32.489072+0000","last_unstale":"2024-09-13T14:39:32.489229+0000","last_undegraded":"2024-09-13T14:39:32.489229+0000","last_fullsized":"2024-09-13T14:39:32.312735+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:38.944519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":324,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:38.546561+0000","last_change":"2024-09-13T14:39:00.143025+0000","last_active":"2024-09-13T14:39:38.546561+0000","last_peered":"2024-09-13T14:39:38.546561+0000","last_clean":"2024-09-13T14:38:58.143730+0000","last_became_active":"2024-09-13T14:39:00.143025+0000","last_became_peered":"2024-09-13T14:39:00.143025+0000","last_unstale":"2024-09-13T14:39:38.546561+0000","last_undegraded":"2024-09-13T14:39:00.140034+0000","last_fullsized":"2024-09-13T14:39:00.139927+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:59.754759+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":505,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331918+0000","last_change":"2024-09-13T14:38:03.598137+0000","last_active":"2024-09-13T14:39:38.331918+0000","last_peered":"2024-09-13T14:39:38.331918+0000","last_clean":"2024-09-13T14:39:38.331918+0000","last_became_active":"2024-09-13T14:38:03.573467+0000","last_became_peered":"2024-09-13T14:38:03.573467+0000","last_unstale":"2024-09-13T14:39:38.331918+0000","last_undegraded":"2024-09-13T14:39:38.331918+0000","last_fullsized":"2024-09-13T14:39:38.331918+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:45.000226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":724,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.320261+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:39:33.320261+0000","last_peered":"2024-09-13T14:39:33.320261+0000","last_clean":"2024-09-13T14:39:33.320261+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:39:33.320261+0000","last_undegraded":"2024-09-13T14:39:33.320261+0000","last_fullsized":"2024-09-13T14:39:33.320261+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":24,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.319712+0000","last_change":"2024-09-13T14:39:00.147458+0000","last_active":"2024-09-13T14:39:33.319712+0000","last_peered":"2024-09-13T14:39:33.319712+0000","last_clean":"2024-09-13T14:13:41.111573+0000","last_became_active":"2024-09-13T14:39:00.147458+0000","last_became_peered":"2024-09-13T14:39:00.147458+0000","last_unstale":"2024-09-13T14:39:33.319712+0000","last_undegraded":"2024-09-13T14:39:33.319712+0000","last_fullsized":"2024-09-13T14:39:00.138377+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:45:08.738571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":270,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.320054+0000","last_change":"2024-09-13T14:39:00.143378+0000","last_active":"2024-09-13T14:39:33.320054+0000","last_peered":"2024-09-13T14:39:33.320054+0000","last_clean":"2024-09-13T14:38:58.142088+0000","last_became_active":"2024-09-13T14:39:00.143378+0000","last_became_peered":"2024-09-13T14:39:00.143378+0000","last_unstale":"2024-09-13T14:39:33.320054+0000","last_undegraded":"2024-09-13T14:39:33.320054+0000","last_fullsized":"2024-09-13T14:39:00.139047+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:30.699256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":105,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.312770+0000","last_change":"2024-09-13T14:38:03.582168+0000","last_active":"2024-09-13T14:39:32.312770+0000","last_peered":"2024-09-13T14:39:32.312770+0000","last_clean":"2024-09-13T14:39:32.312770+0000","last_became_active":"2024-09-13T14:38:03.565923+0000","last_became_peered":"2024-09-13T14:38:03.565923+0000","last_unstale":"2024-09-13T14:39:32.312770+0000","last_undegraded":"2024-09-13T14:39:32.312770+0000","last_fullsized":"2024-09-13T14:39:32.312770+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:43.033859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":433,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545668+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:39:38.545668+0000","last_peered":"2024-09-13T14:39:38.545668+0000","last_clean":"2024-09-13T14:39:38.545668+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:39:38.545668+0000","last_undegraded":"2024-09-13T14:39:38.545668+0000","last_fullsized":"2024-09-13T14:39:38.545668+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":288,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-09-13T14:39:32.313334+0000","last_change":"2024-09-13T14:38:03.620319+0000","last_active":"2024-09-13T14:39:32.313334+0000","last_peered":"2024-09-13T14:39:32.313334+0000","last_clean":"2024-09-13T14:39:32.313334+0000","last_became_active":"2024-09-13T14:38:03.569905+0000","last_became_peered":"2024-09-13T14:38:03.569905+0000","last_unstale":"2024-09-13T14:39:32.313334+0000","last_undegraded":"2024-09-13T14:39:32.313334+0000","last_fullsized":"2024-09-13T14:39:32.313334+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:31:18.469972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":304,"state":"active+undersized","last_fresh":"2024-09-13T14:39:32.312281+0000","last_change":"2024-09-13T14:39:00.142032+0000","last_active":"2024-09-13T14:39:32.312281+0000","last_peered":"2024-09-13T14:39:32.312281+0000","last_clean":"2024-09-13T14:34:04.488710+0000","last_became_active":"2024-09-13T14:39:00.142032+0000","last_became_peered":"2024-09-13T14:39:00.142032+0000","last_unstale":"2024-09-13T14:39:32.312281+0000","last_undegraded":"2024-09-13T14:39:32.312281+0000","last_fullsized":"2024-09-13T14:39:00.140754+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:57.963213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":437,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545021+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:39:38.545021+0000","last_peered":"2024-09-13T14:39:38.545021+0000","last_clean":"2024-09-13T14:39:38.545021+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:39:38.545021+0000","last_undegraded":"2024-09-13T14:39:38.545021+0000","last_fullsized":"2024-09-13T14:39:38.545021+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":482,"reported_epoch":306,"state":"active+undersized","last_fresh":"2024-09-13T14:39:38.332047+0000","last_change":"2024-09-13T14:39:00.152902+0000","last_active":"2024-09-13T14:39:38.332047+0000","last_peered":"2024-09-13T14:39:38.332047+0000","last_clean":"2024-09-13T14:38:58.146249+0000","last_became_active":"2024-09-13T14:39:00.152902+0000","last_became_peered":"2024-09-13T14:39:00.152902+0000","last_unstale":"2024-09-13T14:39:38.332047+0000","last_undegraded":"2024-09-13T14:39:38.332047+0000","last_fullsized":"2024-09-13T14:39:00.147069+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:33:17.234038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":493,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545341+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:39:38.545341+0000","last_peered":"2024-09-13T14:39:38.545341+0000","last_clean":"2024-09-13T14:39:38.545341+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:39:38.545341+0000","last_undegraded":"2024-09-13T14:39:38.545341+0000","last_fullsized":"2024-09-13T14:39:38.545341+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":293,"reported_epoch":305,"state":"active+undersized","last_fresh":"2024-09-13T14:39:33.320186+0000","last_change":"2024-09-13T14:39:00.143275+0000","last_active":"2024-09-13T14:39:33.320186+0000","last_peered":"2024-09-13T14:39:33.320186+0000","last_clean":"2024-09-13T14:38:58.142359+0000","last_became_active":"2024-09-13T14:39:00.143275+0000","last_became_peered":"2024-09-13T14:39:00.143275+0000","last_unstale":"2024-09-13T14:39:33.320186+0000","last_undegraded":"2024-09-13T14:39:33.320186+0000","last_fullsized":"2024-09-13T14:39:00.138765+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:07.710136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":511,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545529+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:39:38.545529+0000","last_peered":"2024-09-13T14:39:38.545529+0000","last_clean":"2024-09-13T14:39:38.545529+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:39:38.545529+0000","last_undegraded":"2024-09-13T14:39:38.545529+0000","last_fullsized":"2024-09-13T14:39:38.545529+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":481,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.332060+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:39:38.332060+0000","last_peered":"2024-09-13T14:39:38.332060+0000","last_clean":"2024-09-13T14:39:38.332060+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:39:38.332060+0000","last_undegraded":"2024-09-13T14:39:38.332060+0000","last_fullsized":"2024-09-13T14:39:38.332060+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":451,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545344+0000","last_change":"2024-09-13T14:38:03.576101+0000","last_active":"2024-09-13T14:39:38.545344+0000","last_peered":"2024-09-13T14:39:38.545344+0000","last_clean":"2024-09-13T14:39:38.545344+0000","last_became_active":"2024-09-13T14:38:03.575676+0000","last_became_peered":"2024-09-13T14:38:03.575676+0000","last_unstale":"2024-09-13T14:39:38.545344+0000","last_undegraded":"2024-09-13T14:39:38.545344+0000","last_fullsized":"2024-09-13T14:39:38.545344+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:19.653722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":500,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546988+0000","last_change":"2024-09-13T14:38:03.572344+0000","last_active":"2024-09-13T14:39:38.546988+0000","last_peered":"2024-09-13T14:39:38.546988+0000","last_clean":"2024-09-13T14:39:38.546988+0000","last_became_active":"2024-09-13T14:38:03.572024+0000","last_became_peered":"2024-09-13T14:38:03.572024+0000","last_unstale":"2024-09-13T14:39:38.546988+0000","last_undegraded":"2024-09-13T14:39:38.546988+0000","last_fullsized":"2024-09-13T14:39:38.546988+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:46.144811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":79,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546451+0000","last_change":"2024-09-13T14:38:03.567837+0000","last_active":"2024-09-13T14:39:38.546451+0000","last_peered":"2024-09-13T14:39:38.546451+0000","last_clean":"2024-09-13T14:39:38.546451+0000","last_became_active":"2024-09-13T14:38:03.567250+0000","last_became_peered":"2024-09-13T14:38:03.567250+0000","last_unstale":"2024-09-13T14:39:38.546451+0000","last_undegraded":"2024-09-13T14:39:38.546451+0000","last_fullsized":"2024-09-13T14:39:38.546451+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:46.756604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":404,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546944+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:39:38.546944+0000","last_peered":"2024-09-13T14:39:38.546944+0000","last_clean":"2024-09-13T14:39:38.546944+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:39:38.546944+0000","last_undegraded":"2024-09-13T14:39:38.546944+0000","last_fullsized":"2024-09-13T14:39:38.546944+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":478,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.331228+0000","last_change":"2024-09-13T14:38:03.574656+0000","last_active":"2024-09-13T14:39:38.331228+0000","last_peered":"2024-09-13T14:39:38.331228+0000","last_clean":"2024-09-13T14:39:38.331228+0000","last_became_active":"2024-09-13T14:38:03.573983+0000","last_became_peered":"2024-09-13T14:38:03.573983+0000","last_unstale":"2024-09-13T14:39:38.331228+0000","last_undegraded":"2024-09-13T14:39:38.331228+0000","last_fullsized":"2024-09-13T14:39:38.331228+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:45.794632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":313,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.546290+0000","last_change":"2024-09-13T14:38:03.577130+0000","last_active":"2024-09-13T14:39:38.546290+0000","last_peered":"2024-09-13T14:39:38.546290+0000","last_clean":"2024-09-13T14:39:38.546290+0000","last_became_active":"2024-09-13T14:38:03.576499+0000","last_became_peered":"2024-09-13T14:38:03.576499+0000","last_unstale":"2024-09-13T14:39:38.546290+0000","last_undegraded":"2024-09-13T14:39:38.546290+0000","last_fullsized":"2024-09-13T14:39:38.546290+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:15.156436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":41,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:38.546958+0000","last_change":"2024-09-13T14:39:00.143908+0000","last_active":"2024-09-13T14:39:38.546958+0000","last_peered":"2024-09-13T14:39:38.546958+0000","last_clean":"2024-09-13T14:13:43.252210+0000","last_became_active":"2024-09-13T14:39:00.143908+0000","last_became_peered":"2024-09-13T14:39:00.143908+0000","last_unstale":"2024-09-13T14:39:38.546958+0000","last_undegraded":"2024-09-13T14:39:00.140364+0000","last_fullsized":"2024-09-13T14:39:00.139609+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:41.843240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545871+0000","last_change":"2024-09-13T14:38:03.571640+0000","last_active":"2024-09-13T14:39:38.545871+0000","last_peered":"2024-09-13T14:39:38.545871+0000","last_clean":"2024-09-13T14:39:38.545871+0000","last_became_active":"2024-09-13T14:38:03.570363+0000","last_became_peered":"2024-09-13T14:38:03.570363+0000","last_unstale":"2024-09-13T14:39:38.545871+0000","last_undegraded":"2024-09-13T14:39:38.545871+0000","last_fullsized":"2024-09-13T14:39:38.545871+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:12.912639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":31,"reported_epoch":305,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:33.320392+0000","last_change":"2024-09-13T14:39:00.143988+0000","last_active":"2024-09-13T14:39:33.320392+0000","last_peered":"2024-09-13T14:39:33.320392+0000","last_clean":"2024-09-13T14:13:43.252359+0000","last_became_active":"2024-09-13T14:39:00.143988+0000","last_became_peered":"2024-09-13T14:39:00.143988+0000","last_unstale":"2024-09-13T14:39:33.320392+0000","last_undegraded":"2024-09-13T14:39:00.139264+0000","last_fullsized":"2024-09-13T14:39:00.139159+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:03.389420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":503,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545412+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:39:38.545412+0000","last_peered":"2024-09-13T14:39:38.545412+0000","last_clean":"2024-09-13T14:39:38.545412+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:39:38.545412+0000","last_undegraded":"2024-09-13T14:39:38.545412+0000","last_fullsized":"2024-09-13T14:39:38.545412+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":489,"reported_epoch":306,"state":"active+clean","last_fresh":"2024-09-13T14:39:38.545745+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:39:38.545745+0000","last_peered":"2024-09-13T14:39:38.545745+0000","last_clean":"2024-09-13T14:39:38.545745+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:39:38.545745+0000","last_undegraded":"2024-09-13T14:39:38.545745+0000","last_fullsized":"2024-09-13T14:39:38.545745+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-09-13T14:39:33.319533+0000","last_change":"2024-09-13T14:38:03.562609+0000","last_active":"2024-09-13T14:39:33.319533+0000","last_peered":"2024-09-13T14:39:33.319533+0000","last_clean":"2024-09-13T14:39:33.319533+0000","last_became_active":"2024-09-13T14:38:03.562291+0000","last_became_peered":"2024-09-13T14:38:03.562291+0000","last_unstale":"2024-09-13T14:39:33.319533+0000","last_undegraded":"2024-09-13T14:39:33.319533+0000","last_fullsized":"2024-09-13T14:39:33.319533+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:49:15.128621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":487,"reported_epoch":306,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:39:38.331850+0000","last_change":"2024-09-13T14:39:00.153920+0000","last_active":"2024-09-13T14:39:38.331850+0000","last_peered":"2024-09-13T14:39:38.331850+0000","last_clean":"2024-09-13T14:38:58.146157+0000","last_became_active":"2024-09-13T14:39:00.153920+0000","last_became_peered":"2024-09-13T14:39:00.153920+0000","last_unstale":"2024-09-13T14:39:38.331850+0000","last_undegraded":"2024-09-13T14:39:00.150520+0000","last_fullsized":"2024-09-13T14:39:00.150416+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:38.314683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1007,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1507,"num_write_kb":96448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":325910528,"data_stored":325910528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1507,"ondisk_log_size":1507,"up":40,"acting":40,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4890,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18209,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":84,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":22,"acting":22,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":542359,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":272,"num_write":54,"num_write_kb":592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":214679,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":27,"acting":27,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397842,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":606376,"kb_used_data":8148,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":93138776,"statfs":{"total":95995035648,"available":95374106624,"internally_reserved":0,"allocated":8343552,"data_stored":6834872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594071,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":453916,"kb_used_data":152404,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93291236,"statfs":{"total":95995035648,"available":95530225664,"internally_reserved":0,"allocated":156061696,"data_stored":154561030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790292,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":857396,"kb_used_data":267436,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":92887756,"statfs":{"total":95995035648,"available":95117062144,"internally_reserved":0,"allocated":273854464,"data_stored":272412194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":133143986528,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697086,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645848,"kb_used_data":4300,"kb_used_omap":4,"kb_used_meta":641531,"kb_avail":93099304,"statfs":{"total":95995035648,"available":95333687296,"internally_reserved":0,"allocated":4403200,"data_stored":2907072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":656927819},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860613,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":641252,"kb_used_data":110236,"kb_used_omap":8,"kb_used_meta":530999,"kb_avail":93103900,"statfs":{"total":95995035648,"available":95338393600,"internally_reserved":0,"allocated":112881664,"data_stored":111374418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":543743820},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020284,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161240,"kb_used_data":7820,"kb_used_omap":9,"kb_used_meta":153398,"kb_avail":93583912,"statfs":{"total":95995035648,"available":95829925888,"internally_reserved":0,"allocated":8007680,"data_stored":6510959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":157080389},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187674,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":554456,"kb_used_data":7820,"kb_used_omap":7,"kb_used_meta":546616,"kb_avail":93190696,"statfs":{"total":95995035648,"available":95427272704,"internally_reserved":0,"allocated":8007680,"data_stored":6511293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":559735115},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":130940928,"data_stored":130940928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":194969600,"data_stored":194969600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:39:47.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:39:47.162 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:47.186 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:47.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:47.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:47.378 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:47] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-09-13T14:39:47.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:47.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:47.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:47.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 19712 19696 175.779 156.188 0.00804989 0.00568295 2024-09-13T14:39:47.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:47.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:47.691 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:47.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:47.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:47.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:47.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:47.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:48.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:48.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:48.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:48.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:48.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:48.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:48.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:48.410 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:39:48.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:48.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:48.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 21698 21682 169.323 124.125 0.00289209 0.00589885 2024-09-13T14:39:48.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:48.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[103001]: pgmap v875: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 713 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 5535/34080 objects degraded (16.241%) 2024-09-13T14:39:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/712883973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[107556]: pgmap v875: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 713 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 5535/34080 objects degraded (16.241%) 2024-09-13T14:39:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/712883973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:48 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:48 smithi189 ceph-mon[90992]: pgmap v875: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 713 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 5535/34080 objects degraded (16.241%) 2024-09-13T14:39:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/712883973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:48 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:39:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:48.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:48.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:48.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:48.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:48.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:49.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:49.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:49.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:49.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:49.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:49.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:49.405 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:49.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:49.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:49.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 23932 23916 166.023 139.625 0.00591664 0.00601781 2024-09-13T14:39:49.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:49.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:49.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[103001]: osdmap e307: 8 total, 7 up, 7 in 2024-09-13T14:39:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5535/34080 objects degraded (16.241%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:49.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[107556]: osdmap e307: 8 total, 7 up, 7 in 2024-09-13T14:39:49.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5535/34080 objects degraded (16.241%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:49 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:39:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:49 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:39:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:49 smithi189 ceph-mon[90992]: osdmap e307: 8 total, 7 up, 7 in 2024-09-13T14:39:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5535/34080 objects degraded (16.241%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:49.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:49.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:49.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:49.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:49.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:50.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:50.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:50.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:50.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:50.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:50.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:50 smithi189 ceph-mon[90992]: pgmap v877: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 8656/52875 objects degraded (16.371%) 2024-09-13T14:39:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:39:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:50.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:50.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 25639 25623 160.09 106.688 0.00336799 0.00624053 2024-09-13T14:39:50.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:50.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:50 smithi017 ceph-mon[107556]: pgmap v877: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 8656/52875 objects degraded (16.371%) 2024-09-13T14:39:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:50 smithi017 ceph-mon[103001]: pgmap v877: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 8656/52875 objects degraded (16.371%) 2024-09-13T14:39:50.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:50.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:50.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:50.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:50.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:51.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:51.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:51.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:51.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:51.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:51.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:51.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:51.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:51.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:51.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 27353 27337 155.275 107.125 0.00219336 0.00640915 2024-09-13T14:39:51.560 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:51.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:51.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:51.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:51.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:51.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:51.954 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:51.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:52.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:52.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:52.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:52.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:52.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:52.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:52.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:52.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:52.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:52.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 15 28879 28864 150.288 95.4375 0.00771451 0.00664714 2024-09-13T14:39:52.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:52.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:52.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:52 smithi017 ceph-mon[103001]: pgmap v878: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 11459/69675 objects degraded (16.446%) 2024-09-13T14:39:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:52 smithi017 ceph-mon[107556]: pgmap v878: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 11459/69675 objects degraded (16.446%) 2024-09-13T14:39:52.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:52.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:52 smithi189 ceph-mon[90992]: pgmap v878: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 11459/69675 objects degraded (16.446%) 2024-09-13T14:39:52.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:52.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:52.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:52.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:52.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:53.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:53.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:53.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:53.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:53.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:53.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:53.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:53.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 30369 30353 145.887 93.0625 0.00329823 0.00684991 2024-09-13T14:39:53.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:53.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:53.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:53.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:53.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:53.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:53.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:54.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:54.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:54.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:54.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:54.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:54.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 32108 32092 143.229 108.688 0.00525536 0.00697571 2024-09-13T14:39:54.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:54.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:54 smithi017 ceph-mon[103001]: pgmap v879: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s; 12569/74601 objects degraded (16.848%) 2024-09-13T14:39:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 12569/74601 objects degraded (16.848%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:54 smithi017 ceph-mon[107556]: pgmap v879: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s; 12569/74601 objects degraded (16.848%) 2024-09-13T14:39:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 12569/74601 objects degraded (16.848%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:54.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:54 smithi189 ceph-mon[90992]: pgmap v879: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 146 MiB/s wr, 2.33k op/s; 12569/74601 objects degraded (16.848%) 2024-09-13T14:39:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 12569/74601 objects degraded (16.848%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:54.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:54.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:54.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:54.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:55.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:55.030 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:55.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:55.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:55.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:55.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:55.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:55.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:55.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:55.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:55.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 33720 33704 140.397 100.75 0.00216261 0.00711554 2024-09-13T14:39:55.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:55.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:55.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:55.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:55.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:55.944 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:55.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:55.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:56.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:56.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:56.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:56.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:56.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:56.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:56.360 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:56.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:56.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:56.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:56.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 35137 35121 137.158 88.5625 0.00902496 0.00728486 2024-09-13T14:39:56.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:56.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:56 smithi017 ceph-mon[103001]: pgmap v880: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s; 14703/88959 objects degraded (16.528%) 2024-09-13T14:39:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:56 smithi017 ceph-mon[107556]: pgmap v880: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s; 14703/88959 objects degraded (16.528%) 2024-09-13T14:39:56.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:56.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:56.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:56 smithi189 ceph-mon[90992]: pgmap v880: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s; 14703/88959 objects degraded (16.528%) 2024-09-13T14:39:56.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:56.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:56.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:56.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:57.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:57.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:57.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:57.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:57.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:57.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:57.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:57.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:57.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:57.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:57.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:57.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 36793 36777 135.177 103.5 0.0541497 0.00738964 2024-09-13T14:39:57.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:57.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:57.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:39:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:39:57] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-09-13T14:39:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:57.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:57.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:57.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:57.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:57.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:57.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:58.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:58.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:58.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:58.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:58.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:58.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:58.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:58.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:58.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:58.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:58.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 38511 38495 133.633 107.375 0.0101044 0.0074779 2024-09-13T14:39:58.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:39:58.586 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:58.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:39:58.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:39:58.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:58.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:39:58.734 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:58 smithi017 ceph-mon[103001]: pgmap v881: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s; 15709/94929 objects degraded (16.548%) 2024-09-13T14:39:58.735 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:58 smithi017 ceph-mon[107556]: pgmap v881: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s; 15709/94929 objects degraded (16.548%) 2024-09-13T14:39:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:58 smithi189 ceph-mon[90992]: pgmap v881: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 1.9 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s; 15709/94929 objects degraded (16.548%) 2024-09-13T14:39:58.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:58.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:58.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:58.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:59.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:39:59.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:59.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:59.232 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:59.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:59.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:59.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:39:59.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:39:59.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:39:59.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 40347 40331 132.638 114.75 0.00602265 0.00753361 2024-09-13T14:39:59.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:39:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:39:59.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:39:59.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:39:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:39:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 15709/94929 objects degraded (16.548%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:39:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 15709/94929 objects degraded (16.548%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:39:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 15709/94929 objects degraded (16.548%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:59.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:39:59.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:39:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:39:59.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:39:59.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:39:59.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:00.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:00.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:00.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:00.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:00.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:00.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:00.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:00.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:00.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:40:00.553786+0000 min lat: 0.00122954 max lat: 0.290768 avg lat: 0.00758842 2024-09-13T14:40:00.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:00.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 42164 42148 131.684 113.562 0.00603627 0.00758842 2024-09-13T14:40:00.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:00.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:00.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:40:00.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:00.821 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pgmap v882: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s; 17709/107115 objects degraded (16.533%) 2024-09-13T14:40:00.821 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: osd.5 (root=default,host=smithi189) is down 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: [WRN] PG_DEGRADED: Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 1.1 is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 1.4 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 2.3 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 3.1c is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 3.1d is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.2 is active+undersized+degraded, acting [0,3] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.4 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.9 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:00 smithi189 ceph-mon[90992]: pg 8.d is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pgmap v882: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s; 17709/107115 objects degraded (16.533%) 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: osd.5 (root=default,host=smithi189) is down 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: [WRN] PG_DEGRADED: Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 1.1 is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 1.4 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 2.3 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 3.1c is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 3.1d is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.2 is active+undersized+degraded, acting [0,3] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.4 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.9 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[103001]: pg 8.d is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pgmap v882: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s; 17709/107115 objects degraded (16.533%) 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: osd.5 (root=default,host=smithi189) is down 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: [WRN] PG_DEGRADED: Degraded data redundancy: 17709/107115 objects degraded (16.533%), 17 pgs degraded 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 1.1 is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 1.4 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 2.3 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 3.1c is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 3.1d is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.0 is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.2 is active+undersized+degraded, acting [0,3] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.3 is active+undersized+degraded, acting [1,6] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.4 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.8 is active+undersized+degraded, acting [3,7] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.9 is active+undersized+degraded, acting [6,0] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.a is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:00 smithi017 ceph-mon[107556]: pg 8.d is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:00.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:00.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:01.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:01.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:01.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:01.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:01.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:01.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:01.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:01.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:01.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:01.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:01.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 43697 43681 129.958 95.8125 0.0143634 0.00768746 2024-09-13T14:40:01.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:01.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:01.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:01.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:01.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:01.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:01.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:01 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:01.935 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:01 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:01.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:02.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:01 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:02.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44494 8.a 8:5871664e:::benchmark_data_smithi017_253885_object44493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.106382+0000", 2024-09-13T14:40:02.119 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.004536801, 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045772030000000002, 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44494 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106382+0000", 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106382+0000", 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106383+0000", 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4440000000000001e-06 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106408+0000", 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4918000000000001e-05 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106418+0000", 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0206e-05 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106425+0000", 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8839999999999997e-06 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106502+0000", 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7231000000000006e-05 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106633+0000", 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013077499999999999 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106777+0000", 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144022 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44495 8.a 8:59d03d47:::benchmark_data_smithi017_253885_object44494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.106445+0000", 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0044730229999999996, 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046066620000000001, 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44495 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106445+0000", 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106445+0000", 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106446+0000", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5000000000000003e-07 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106496+0000", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0195000000000003e-05 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106505+0000", 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9269999999999994e-06 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106512+0000", 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5180000000000001e-06 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.108657+0000", 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002145564 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.108768+0000", 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011041599999999999 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.109758+0000", 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0009903030000000001 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.129 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44488 8.7 e307/303 ondisk, result = 0)", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.106741+0000", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0041768700000000001, 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043775910000000001, 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106741+0000", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106741+0000", 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106742+0000", 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0500000000000003e-07 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106746+0000", 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.766e-06 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106754+0000", 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7039999999999994e-06 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106758+0000", 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5940000000000004e-06 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111097+0000", 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043387640000000002 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111105+0000", 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7810000000000005e-06 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:40:02.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111119+0000", 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4377e-05 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44475 8.7 8:e20c31d0:::benchmark_data_smithi017_253885_object44474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.090312+0000", 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.020606342999999999, 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020891249000000001, 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.134 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44475 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090312+0000", 2024-09-13T14:40:02.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090312+0000", 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090312+0000", 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090356+0000", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3420000000000001e-05 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090364+0000", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8730000000000007e-06 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.090369+0000", 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4890000000000003e-06 2024-09-13T14:40:02.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.091214+0000", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084465600000000005 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.091313+0000", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8895999999999996e-05 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.091429+0000", 2024-09-13T14:40:02.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116297 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.098293+0000", 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00686391 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111171+0000", 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012878459 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44497 8.7 8.5126ca57 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.107028+0000", 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0038905570000000002, 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042567339999999999, 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.140 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44497 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107028+0000", 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107028+0000", 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107029+0000", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3900000000000004e-07 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107075+0000", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6660999999999999e-05 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107087+0000", 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1343e-05 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107094+0000", 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4059999999999999e-06 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44498 8.a 8:54748dea:::benchmark_data_smithi017_253885_object44497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.107547+0000", 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0033713060000000001, 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038064319999999998, 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.143 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44498 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107547+0000", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107547+0000", 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107548+0000", 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0999999999999997e-07 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107594+0000", 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6375000000000002e-05 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107609+0000", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4800999999999999e-05 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.107616+0000", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.002e-06 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111269+0000", 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036528440000000001 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.146 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44477 8.7 8:e2347079:::benchmark_data_smithi017_253885_object44476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.092524+0000", 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.018394232999999999, 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018883282000000001, 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44477 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092524+0000", 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092524+0000", 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092525+0000", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9500000000000003e-07 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092572+0000", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7617000000000002e-05 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092580+0000", 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0590000000000003e-06 2024-09-13T14:40:02.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.092586+0000", 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5040000000000002e-06 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.095862+0000", 2024-09-13T14:40:02.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003276073 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.096207+0000", 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034539599999999998 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.096372+0000", 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016465699999999999 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.098336+0000", 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019643389999999998 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111177+0000", 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012840786 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44475 8.7 e307/303 ondisk, result = 0)", 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.110442+0000", 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00047646199999999998, 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010522120000000001, 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110442+0000", 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110442+0000", 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110443+0000", 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.99e-07 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110446+0000", 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7610000000000001e-06 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110463+0000", 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.615e-05 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110469+0000", 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6839999999999999e-06 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111210+0000", 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00074125700000000005 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111217+0000", 2024-09-13T14:40:02.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8920000000000003e-06 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44492 8.7 e307/303)", 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.110715+0000", 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.000203654, 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082879799999999995, 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.157 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110715+0000", 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110715+0000", 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110715+0000", 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3e-07 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110719+0000", 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1640000000000003e-06 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110729+0000", 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4360000000000008e-06 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.110736+0000", 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.917e-06 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.160 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44477 8.7 e307/303)", 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.111345+0000", 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9995732, 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000237233, 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111345+0000", 2024-09-13T14:40:02.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111345+0000", 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111346+0000", 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8100000000000003e-07 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111350+0000", 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0969999999999999e-06 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111360+0000", 2024-09-13T14:40:02.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8260000000000005e-06 2024-09-13T14:40:02.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111366+0000", 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6120000000000004e-06 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:44501 8.0 e307/303 8:025e9b8d:::benchmark_data_smithi017_253885_object44500:head v 307'2839, mlcod=307'2836)", 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.111391+0000", 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9995275, 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023275999999999999, 2024-09-13T14:40:02.168 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111391+0000", 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111391+0000", 2024-09-13T14:40:02.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111391+0000", 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4799999999999999e-07 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111416+0000", 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4071e-05 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111424+0000", 2024-09-13T14:40:02.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2779999999999993e-06 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111429+0000", 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5459999999999999e-06 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111471+0000", 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2154999999999999e-05 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111492+0000", 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0018999999999999e-05 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44483 8.7 e307/303)", 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.111446+0000", 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9994726, 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022640900000000001, 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111446+0000", 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111446+0000", 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111446+0000", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3200000000000001e-07 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111449+0000", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3949999999999999e-06 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111455+0000", 2024-09-13T14:40:02.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6719999999999998e-06 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111460+0000", 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7180000000000004e-06 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.181 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44483 8.7 8:ec460bbf:::benchmark_data_smithi017_253885_object44482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.097004+0000", 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.01391454, 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014716468, 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44483 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097004+0000", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097004+0000", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097005+0000", 2024-09-13T14:40:02.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4e-07 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097051+0000", 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5884999999999999e-05 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097062+0000", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1547e-05 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.097069+0000", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.72e-06 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099191+0000", 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021223000000000001 2024-09-13T14:40:02.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099309+0000", 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011798699999999999 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099438+0000", 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012846 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.104341+0000", 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049031709999999996 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111182+0000", 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068411399999999999 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44494 8.a e307/303)", 2024-09-13T14:40:02.187 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.111458+0000", 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9994607, 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034100299999999999, 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111458+0000", 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111458+0000", 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111458+0000", 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8700000000000005e-07 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111463+0000", 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7609999999999998e-06 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111473+0000", 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5719999999999992e-06 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111479+0000", 2024-09-13T14:40:02.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2539999999999998e-06 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:44486 8.7 e307/303)", 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.111468+0000", 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9994507, 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036968, 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-13T14:40:02.191 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111468+0000", 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111468+0000", 2024-09-13T14:40:02.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111468+0000", 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7099999999999998e-07 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111470+0000", 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.198e-06 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111476+0000", 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6330000000000001e-06 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111480+0000", 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5750000000000002e-06 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44486 8.7 8:e906ec81:::benchmark_data_smithi017_253885_object44485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.098968+0000", 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011949905, 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012928014999999999, 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44486 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.195 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.098968+0000", 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.098968+0000", 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.098969+0000", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8899999999999999e-07 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099016+0000", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6768999999999999e-05 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099025+0000", 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0129999999999999e-06 2024-09-13T14:40:02.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099031+0000", 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4459999999999998e-06 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101249+0000", 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022178050000000002 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101362+0000", 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112664 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101495+0000", 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013315199999999999 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.104381+0000", 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002885908 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111553+0000", 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071723250000000002 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44488 8.7 8:e320df52:::benchmark_data_smithi017_253885_object44487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.099359+0000", 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.01155923, 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012616382000000001, 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.200 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44488 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099359+0000", 2024-09-13T14:40:02.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099359+0000", 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099360+0000", 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6300000000000004e-07 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099407+0000", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6628999999999997e-05 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099416+0000", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4579999999999999e-06 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.099424+0000", 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6590000000000007e-06 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.104410+0000", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049860679999999998 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.104521+0000", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111217 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.104659+0000", 2024-09-13T14:40:02.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013778 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111111+0000", 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064525659999999999 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111559+0000", 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044724000000000001 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44489 8.7 8:e4abd295:::benchmark_data_smithi017_253885_object44488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.101734+0000", 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0091842820000000002, 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010322382, 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44489 2024-09-13T14:40:02.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101734+0000", 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101734+0000", 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101736+0000", 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6759999999999999e-06 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101791+0000", 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4758e-05 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.101802+0000", 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1039000000000001e-05 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.102295+0000", 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049345400000000005 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106207+0000", 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039121939999999999 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106397+0000", 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018946300000000001 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.106533+0000", 2024-09-13T14:40:02.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013661599999999999 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111158+0000", 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046249280000000004 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111563+0000", 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040496999999999999 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:44492 8.7 8:eb8e1ac7:::benchmark_data_smithi017_253885_object44491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2024-09-13T14:40:02.213 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:02.103708+0000", 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0072100549999999999, 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084252830000000004, 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:40:02.214 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 44492 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103708+0000", 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103708+0000", 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103709+0000", 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.062e-06 2024-09-13T14:40:02.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103730+0000", 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0227000000000001e-05 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103746+0000", 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6147e-05 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.103753+0000", 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0539999999999997e-06 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.108416+0000", 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046631620000000002 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:40:02.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.109232+0000", 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00081647200000000005 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.109373+0000", 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014018400000000001 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111657+0000", 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022841490000000001 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:02.111671+0000", 2024-09-13T14:40:02.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4684999999999999e-05 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 19 2024-09-13T14:40:02.222 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:40:02.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:02.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:02.224 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:40:02.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:02.226 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:40:02.227 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:02.227 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:02.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:02.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:02.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:02.482 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:02.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 45230 45214 128.406 95.8125 0.0120476 0.00778218 2024-09-13T14:40:02.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:02.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:02.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:02 smithi189 ceph-mon[90992]: pgmap v883: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s; 19816/119580 objects degraded (16.571%) 2024-09-13T14:40:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:02 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:02 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3732649314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:02.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:02.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:02.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:02.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[103001]: pgmap v883: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s; 19816/119580 objects degraded (16.571%) 2024-09-13T14:40:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3732649314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[107556]: pgmap v883: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s; 19816/119580 objects degraded (16.571%) 2024-09-13T14:40:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:02 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3732649314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:02.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:03.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:03.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:03.162 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:03.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:03.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:03.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:03.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:03.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:03.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:03.553 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:03.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 46803 46787 127.098 98.3125 0.0097975 0.00786315 2024-09-13T14:40:03.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:03.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:03.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:03.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:03.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:03.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:03 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 19816/119580 objects degraded (16.571%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:03 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 19816/119580 objects degraded (16.571%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:03.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:03.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:04.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:03 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 19816/119580 objects degraded (16.571%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:04.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:04.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:04.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:04.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:04.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:04.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:04.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:04.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:04.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 48319 48303 125.75 94.75 0.00472449 0.00794739 2024-09-13T14:40:04.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:04.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:04.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:04.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:04.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:04.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:04.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:04.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:04 smithi017 ceph-mon[103001]: pgmap v884: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 20835/124254 objects degraded (16.768%) 2024-09-13T14:40:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:04 smithi017 ceph-mon[107556]: pgmap v884: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 20835/124254 objects degraded (16.768%) 2024-09-13T14:40:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:04 smithi189 ceph-mon[90992]: pgmap v884: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 20835/124254 objects degraded (16.768%) 2024-09-13T14:40:05.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:05.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:05.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:05.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:05.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:05.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:05.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:05.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 49624 49608 123.982 81.5625 0.00699696 0.00805994 2024-09-13T14:40:05.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:05.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:05.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:05.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:05.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:05.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:05.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:05.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:05.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:06.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:06.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:06.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:06.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:06.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:06.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:06.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:06.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:06.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:06.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:06.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 51089 51073 122.735 91.5625 0.0146741 0.00814112 2024-09-13T14:40:06.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:06.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:40:06.700 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:40:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:06.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:06 smithi189 ceph-mon[90992]: pgmap v885: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 22860/138075 objects degraded (16.556%) 2024-09-13T14:40:06.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:06 smithi017 ceph-mon[103001]: pgmap v885: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 22860/138075 objects degraded (16.556%) 2024-09-13T14:40:06.787 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:06 smithi017 ceph-mon[107556]: pgmap v885: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 22860/138075 objects degraded (16.556%) 2024-09-13T14:40:06.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:06.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:06.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:06.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:07.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:07.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:07.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:07.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:07.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:07.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:07.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:07.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:07] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-09-13T14:40:07.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:07.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 52473 52457 121.393 86.5 0.00733387 0.00823147 2024-09-13T14:40:07.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:07.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:07.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:40:07.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:07.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:07.735 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-13T14:40:07.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:07.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:07.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[103001]: pgmap v886: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 23755/143670 objects degraded (16.534%) 2024-09-13T14:40:07.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:07.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:07.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:07.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[107556]: pgmap v886: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 23755/143670 objects degraded (16.534%) 2024-09-13T14:40:07.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:07.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:07 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:07 smithi189 ceph-mon[90992]: pgmap v886: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 23755/143670 objects degraded (16.534%) 2024-09-13T14:40:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:40:08.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:08.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:08.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:08.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:08.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:08.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:08.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:08.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:08.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 53814 53798 120.051 83.8125 0.0209637 0.00832422 2024-09-13T14:40:08.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:08.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:08.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:08.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:40:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:08.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:08.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:08.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:08.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:08.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:08.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:08 smithi189 ceph-mon[90992]: osdmap e308: 8 total, 7 up, 7 in 2024-09-13T14:40:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:08 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 23755/143670 objects degraded (16.534%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:09.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:09.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[103001]: osdmap e308: 8 total, 7 up, 7 in 2024-09-13T14:40:09.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 23755/143670 objects degraded (16.534%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:09.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3864028301' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:40:09.050 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[107556]: osdmap e308: 8 total, 7 up, 7 in 2024-09-13T14:40:09.050 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:08 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 23755/143670 objects degraded (16.534%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:09.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:09.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:09.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:09.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:09.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:09.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 55255 55239 119.017 90.0625 0.0178223 0.00839654 2024-09-13T14:40:09.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:09.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:09.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:09.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:09.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:09.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:09.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:09 smithi189 ceph-mon[90992]: pgmap v888: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 25539/154209 objects degraded (16.561%) 2024-09-13T14:40:10.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:10.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:10.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:09 smithi017 ceph-mon[103001]: pgmap v888: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 25539/154209 objects degraded (16.561%) 2024-09-13T14:40:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:09 smithi017 ceph-mon[107556]: pgmap v888: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 25539/154209 objects degraded (16.561%) 2024-09-13T14:40:10.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:10.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:10.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:10.394 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:10.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:10.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:10.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 56707 56691 118.073 90.75 0.00999954 0.00846389 2024-09-13T14:40:10.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:10.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:10.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:10.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:10.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:10.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:10.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:11.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:11.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:11.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:11.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:11.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:11.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:11.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:11.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 58047 58031 116.966 83.75 0.0107372 0.00854457 2024-09-13T14:40:11.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:11.607 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:11.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:11.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:11.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:11.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:11.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:11.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:12.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:12.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:12.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:12.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:12.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:12.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:12 smithi017 ceph-mon[107556]: pgmap v889: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 27349/164934 objects degraded (16.582%) 2024-09-13T14:40:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:12 smithi017 ceph-mon[103001]: pgmap v889: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 27349/164934 objects degraded (16.582%) 2024-09-13T14:40:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:12 smithi189 ceph-mon[90992]: pgmap v889: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 27349/164934 objects degraded (16.582%) 2024-09-13T14:40:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:12.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:12.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 59229 59213 115.619 73.875 0.0247077 0.00864277 2024-09-13T14:40:12.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:12.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:12.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:12.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:12.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:12.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:12.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:12.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:13.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:13.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:13.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:13.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:13.230 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:40:13.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:13.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:13 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1583795998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:13 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:13 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1583795998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:13 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:13.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:13.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:13 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1583795998' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:13 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:40:13.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 60571 60555 114.657 83.875 0.00778444 0.00871622 2024-09-13T14:40:13.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:13.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:13.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:13.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:13.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:13.874 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:13.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:13.877 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2024-09-13T14:40:13.877 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:40:13.877 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-09-13T14:40:13.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd in 4 2024-09-13T14:40:13.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:14.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:14.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:14.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:14.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:14.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:14.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[103001]: pgmap v890: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 28622/170025 objects degraded (16.834%) 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[103001]: osdmap e309: 8 total, 7 up, 7 in 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 28622/170025 objects degraded (16.834%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[107556]: pgmap v890: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 28622/170025 objects degraded (16.834%) 2024-09-13T14:40:14.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[107556]: osdmap e309: 8 total, 7 up, 7 in 2024-09-13T14:40:14.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 28622/170025 objects degraded (16.834%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:14.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:14 smithi189 ceph-mon[90992]: pgmap v890: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 28622/170025 objects degraded (16.834%) 2024-09-13T14:40:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:14 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:14 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:40:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:14 smithi189 ceph-mon[90992]: osdmap e309: 8 total, 7 up, 7 in 2024-09-13T14:40:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 28622/170025 objects degraded (16.834%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:14.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 61743 61727 113.44 73.25 0.0102864 0.00880893 2024-09-13T14:40:14.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:14.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:14.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:14.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:14.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:14.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:14.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:14.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:15.075 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:15.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:15.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:15.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:15.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:15.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:15.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:15.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:15.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:15.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 63084 63068 112.593 83.8125 0.0274939 0.00887598 2024-09-13T14:40:15.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:15.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:15.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:15.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:15.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:15.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:15.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:16.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:16.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:16.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:16.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:16.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:16.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:16 smithi017 ceph-mon[103001]: pgmap v892: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 29810/179913 objects degraded (16.569%) 2024-09-13T14:40:16.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:16 smithi017 ceph-mon[107556]: pgmap v892: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 29810/179913 objects degraded (16.569%) 2024-09-13T14:40:16.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:16 smithi189 ceph-mon[90992]: pgmap v892: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s; 29810/179913 objects degraded (16.569%) 2024-09-13T14:40:16.519 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:16.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 64459 64443 111.852 85.9375 0.0101246 0.0089351 2024-09-13T14:40:16.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:16.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:16.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:16.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:16.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:16.926 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:16.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:17.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:17.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:40:17.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:17.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:17.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:17.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:17.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:17.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 65737 65721 110.988 79.875 0.0193008 0.00900462 2024-09-13T14:40:17.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:17.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:17] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.51.0" 2024-09-13T14:40:17.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:17.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:17.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:17.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:17.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:18.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:18.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:18.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:18.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:18.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:18.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:18.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:18 smithi017 ceph-mon[103001]: pgmap v893: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 30644/184656 objects degraded (16.595%) 2024-09-13T14:40:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:18 smithi017 ceph-mon[107556]: pgmap v893: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 30644/184656 objects degraded (16.595%) 2024-09-13T14:40:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:18.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:18 smithi189 ceph-mon[90992]: pgmap v893: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 30644/184656 objects degraded (16.595%) 2024-09-13T14:40:18.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 67022 67006 110.181 80.3125 0.0111383 0.00907075 2024-09-13T14:40:18.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:18.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:18.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:18.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:18.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:18.818 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:18.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:18.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:18.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:19.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:19.027 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:19.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:19.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:19.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:19.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:19.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:19.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:19.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 30644/184656 objects degraded (16.595%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:19.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 30644/184656 objects degraded (16.595%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 30644/184656 objects degraded (16.595%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:19.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:19.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:19.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 68264 68248 109.346 77.625 0.0151258 0.00914025 2024-09-13T14:40:19.618 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:40:19.618 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:40:19.618 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:40:19.619 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:40:19.619 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:40:19.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:19.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:19.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:19.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:19.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:19.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:19.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:20.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:20.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:20.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:20.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:20.334 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:20.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:20 smithi017 ceph-mon[103001]: pgmap v894: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 32233/194343 objects degraded (16.586%) 2024-09-13T14:40:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:20 smithi017 ceph-mon[107556]: pgmap v894: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 32233/194343 objects degraded (16.586%) 2024-09-13T14:40:20.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:20.482 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:20 smithi189 ceph-mon[90992]: pgmap v894: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 32233/194343 objects degraded (16.586%) 2024-09-13T14:40:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:20.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:40:20.558845+0000 min lat: 0.00122954 max lat: 0.290768 avg lat: 0.00920378 2024-09-13T14:40:20.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:20.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 69529 69513 108.589 79.0625 0.00724317 0.00920378 2024-09-13T14:40:20.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:20.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:20.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:20.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:20.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:40:20.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:20.916 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:20.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:20.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:21.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:21.047 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:21.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:21.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:21.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:21.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:21.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:21.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:21.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:21.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 70713 70697 107.745 74 0.0122307 0.00927649 2024-09-13T14:40:21.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:21.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:21.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:21.657 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:40:21.657 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:21.693 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:40:21.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:21.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:40:21.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:21.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:21.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:21.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:21.777 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:40:21.777 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:21.806 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:40:21.806 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:21.808 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:40:21.842 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:40:21.842 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:21.873 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:40:21.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:21.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:21.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:21.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:22.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:22.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:22.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:22.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:22.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:22.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:22.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:22 smithi017 ceph-mon[107556]: pgmap v895: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s; 33843/204123 objects degraded (16.580%) 2024-09-13T14:40:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2334121838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:22.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:22 smithi017 ceph-mon[103001]: pgmap v895: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s; 33843/204123 objects degraded (16.580%) 2024-09-13T14:40:22.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2334121838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:22.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:22 smithi189 ceph-mon[90992]: pgmap v895: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s; 33843/204123 objects degraded (16.580%) 2024-09-13T14:40:22.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2334121838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:22.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 71967 71951 107.046 78.375 0.00535071 0.00933642 2024-09-13T14:40:22.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:22.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:22.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:22.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:22.819 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:22.842 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:22.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:22.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:22.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:23.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.104 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:23.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:23.205 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:23.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:23.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:23.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:23.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:23.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:23.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 73224 73208 106.383 78.5625 0.00958136 0.00939517 2024-09-13T14:40:23.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:23.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:23.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:23.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:40:23.821 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:40:23.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:23.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:23.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:24.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:24.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:24.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:24.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:24.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:24.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:24.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:24 smithi017 ceph-mon[103001]: pgmap v896: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34986/208782 objects degraded (16.757%) 2024-09-13T14:40:24.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34986/208782 objects degraded (16.757%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:24.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:24 smithi017 ceph-mon[107556]: pgmap v896: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34986/208782 objects degraded (16.757%) 2024-09-13T14:40:24.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34986/208782 objects degraded (16.757%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:24.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:24.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:24 smithi189 ceph-mon[90992]: pgmap v896: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34986/208782 objects degraded (16.757%) 2024-09-13T14:40:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34986/208782 objects degraded (16.757%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:24.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:24.560 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 74347 74331 105.558 70.1875 0.0292521 0.00946722 2024-09-13T14:40:24.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:24.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:24.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:24.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:24.831 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:24.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:24.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:24.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:24.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:24.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:25.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:25.104 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:25.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:25.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:25.244 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.4. 2024-09-13T14:40:25.245 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:24 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T14:40:24.987+0000 7f0b3f9c1640 -1 mon.a@0(leader).osd e309 definitely_dead 0 2024-09-13T14:40:25.245 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:40:25.245 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:40:25.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:25.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:25.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:25.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:40:25.560 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 75030 75014 104.161 42.6875 0.00346437 0.00953318 2024-09-13T14:40:25.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:25.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:25.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:25.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:25.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:40:25.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:25.854 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:25.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:25.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:25.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:26.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:26.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:26.211 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:26.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:26.246 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:40:26.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:26.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:26.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:26.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[103001]: pgmap v897: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 36078/217920 objects degraded (16.556%) 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[103001]: osdmap e310: 8 total, 7 up, 8 in 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2466760486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[107556]: pgmap v897: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 36078/217920 objects degraded (16.556%) 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[107556]: osdmap e310: 8 total, 7 up, 8 in 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2466760486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:26.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:26 smithi189 ceph-mon[90992]: pgmap v897: 99 pgs: 20 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 36078/217920 objects degraded (16.556%) 2024-09-13T14:40:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2349229709' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:40:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:26 smithi189 ceph-mon[90992]: osdmap e310: 8 total, 7 up, 8 in 2024-09-13T14:40:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2466760486' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:26.560 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 75030 75014 101.897 0 - 0.00953318 2024-09-13T14:40:26.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:26.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:26.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:26.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:26.749 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-09-13T14:40:26.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:26.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:26.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:26.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:26.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:27.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:27.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:27.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:27.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:27.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:27.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:27] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-09-13T14:40:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: osdmap e311: 8 total, 7 up, 8 in 2024-09-13T14:40:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[107556]: osdmap e312: 8 total, 7 up, 8 in 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: osdmap e311: 8 total, 7 up, 8 in 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:27 smithi017 ceph-mon[103001]: osdmap e312: 8 total, 7 up, 8 in 2024-09-13T14:40:27.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:27.473 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:27.524 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:27.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 75081 75065 99.7933 1.59375 0.0107305 0.0100147 2024-09-13T14:40:27.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:27.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:27.643 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:27.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:40:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: osdmap e311: 8 total, 7 up, 8 in 2024-09-13T14:40:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:27.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:27.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:27 smithi189 ceph-mon[90992]: osdmap e312: 8 total, 7 up, 8 in 2024-09-13T14:40:27.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:27.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:27.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:27.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:27.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:28.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:28.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:28.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:28.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:28.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:28.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:28 smithi017 ceph-mon[103001]: pgmap v900: 99 pgs: 1 remapped+peering, 3 peering, 20 active+undersized, 16 active+undersized+degraded, 59 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 32229/222495 objects degraded (14.485%) 2024-09-13T14:40:28.329 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:28 smithi017 ceph-mon[107556]: pgmap v900: 99 pgs: 1 remapped+peering, 3 peering, 20 active+undersized, 16 active+undersized+degraded, 59 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 32229/222495 objects degraded (14.485%) 2024-09-13T14:40:28.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:28.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:28.481 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:28.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:28.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 75977 75961 98.881 56 0.0141217 0.0101078 2024-09-13T14:40:28.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:28.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:28.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:28 smithi189 ceph-mon[90992]: pgmap v900: 99 pgs: 1 remapped+peering, 3 peering, 20 active+undersized, 16 active+undersized+degraded, 59 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 32229/222495 objects degraded (14.485%) 2024-09-13T14:40:28.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:28.881 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:28.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:28.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:29.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:29.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:29.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:29.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:40:29.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:29.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:29.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:29.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:29 smithi017 ceph-mon[103001]: osdmap e313: 8 total, 7 up, 8 in 2024-09-13T14:40:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32229/222495 objects degraded (14.485%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:29 smithi017 ceph-mon[107556]: osdmap e313: 8 total, 7 up, 8 in 2024-09-13T14:40:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32229/222495 objects degraded (14.485%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:29.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:29.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:29.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 76896 76880 98.0352 57.4375 0.0106262 0.0101946 2024-09-13T14:40:29.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:29.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:29 smithi189 ceph-mon[90992]: osdmap e313: 8 total, 7 up, 8 in 2024-09-13T14:40:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32229/222495 objects degraded (14.485%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:29.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:29.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:29.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:29.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:29.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:29.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:30.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:30.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:30.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:30.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:30.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:30.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:30.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:30 smithi189 ceph-mon[90992]: pgmap v903: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 13 active+undersized+degraded, 59 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 42123/226146 objects degraded (18.626%); 4627/226146 objects misplaced (2.046%); 595 KiB/s, 2 keys/s, 14 objects/s recovering 2024-09-13T14:40:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:30 smithi189 ceph-mon[90992]: osdmap e314: 8 total, 7 up, 8 in 2024-09-13T14:40:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:30.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 77905 77889 97.3336 63.0625 0.0151028 0.0102682 2024-09-13T14:40:30.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:30.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:30.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:30 smithi017 ceph-mon[103001]: pgmap v903: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 13 active+undersized+degraded, 59 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 42123/226146 objects degraded (18.626%); 4627/226146 objects misplaced (2.046%); 595 KiB/s, 2 keys/s, 14 objects/s recovering 2024-09-13T14:40:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:30 smithi017 ceph-mon[103001]: osdmap e314: 8 total, 7 up, 8 in 2024-09-13T14:40:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:30 smithi017 ceph-mon[107556]: pgmap v903: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 13 active+undersized+degraded, 59 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 42123/226146 objects degraded (18.626%); 4627/226146 objects misplaced (2.046%); 595 KiB/s, 2 keys/s, 14 objects/s recovering 2024-09-13T14:40:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:30 smithi017 ceph-mon[107556]: osdmap e314: 8 total, 7 up, 8 in 2024-09-13T14:40:30.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:30.761 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:30.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:30.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:30.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:31.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:31.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:31.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:31.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:31.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:31.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:31.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:31.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:31.563 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 78905 78889 96.6507 62.5 0.0189107 0.0103405 2024-09-13T14:40:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:31.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:31.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:31.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:31.750 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2024-09-13T14:40:31.750 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-09-13T14:40:31.750 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-09-13T14:40:31.751 DEBUG:teuthology.orchestra.run.smithi189:osd.5> sudo journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:40:31.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl start ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5 2024-09-13T14:40:31.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:31.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:31.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:31.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:32.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:32.118 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:31 smithi189 systemd[1]: Starting Ceph osd.5 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:40:32.118 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.02287673 +0000 UTC m=+0.028291946 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:40:32.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:32.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:32.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:32.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:32.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:32.442 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.150271216 +0000 UTC m=+0.155686410 container create 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-13T14:40:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:32.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:32.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:32.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 79834 79818 95.9069 58.0625 0.0253291 0.0104209 2024-09-13T14:40:32.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:32.620 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:32.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:32.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:32.756 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.44162421 +0000 UTC m=+0.447039418 container init 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T14:40:32.756 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.450359327 +0000 UTC m=+0.455774524 container start 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T14:40:32.756 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.558226746 +0000 UTC m=+0.563641939 container attach 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:40:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-mon[90992]: pgmap v905: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s; 47601/230295 objects degraded (20.670%); 4627/230295 objects misplaced (2.009%); 486 KiB/s, 2 keys/s, 13 objects/s recovering 2024-09-13T14:40:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[103001]: pgmap v905: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s; 47601/230295 objects degraded (20.670%); 4627/230295 objects misplaced (2.009%); 486 KiB/s, 2 keys/s, 13 objects/s recovering 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[107556]: pgmap v905: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s; 47601/230295 objects degraded (20.670%); 4627/230295 objects misplaced (2.009%); 486 KiB/s, 2 keys/s, 13 objects/s recovering 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:32.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:32.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:33.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:33.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:33.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:33.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:33.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:40:33.197 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate[171913]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 bash[171900]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T14:40:33.198 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:32 smithi189 podman[171900]: 2024-09-13 14:40:32.864518052 +0000 UTC m=+0.869933248 container died 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2) 2024-09-13T14:40:33.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:33.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:33.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:33.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:33.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:33.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:33.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 80683 80667 95.0986 53.0625 0.00951738 0.0105096 2024-09-13T14:40:33.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:33.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:33.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 47601/230295 objects degraded (20.670%), 17 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:33 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 47601/230295 objects degraded (20.670%), 17 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:33.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:33.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:33.798 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:33 smithi189 podman[172078]: 2024-09-13 14:40:33.505333904 +0000 UTC m=+0.632880497 container remove 403ecf40800eb5f0ae3c865dbb7e21e27a17bd3663d52a951fc35d99132e453f (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:40:33.799 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:33 smithi189 podman[172221]: 2024-09-13 14:40:33.65826773 +0000 UTC m=+0.026750652 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:40:33.799 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.799 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.799 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:33.799 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:33 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 47601/230295 objects degraded (20.670%), 17 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:33.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:33.882 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:33.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:33.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:34.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:34.049 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:40:34.049 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:34.072 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:40:34.072 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:40:34.085 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:33 smithi189 podman[172221]: 2024-09-13 14:40:33.798280207 +0000 UTC m=+0.166763127 container create c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T14:40:34.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:40:34.086 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:34.124 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:40:34.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:34.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:34.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:40:34.182 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:34.200 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:40:34.217 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:40:34.218 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:34.218 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-09-13T14:40:34.248 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:40:34.249 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:34.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:34.283 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:40:34.284 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:34.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:34.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:34.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 podman[172221]: 2024-09-13 14:40:34.085374039 +0000 UTC m=+0.453856955 container init c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:40:34.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 podman[172221]: 2024-09-13 14:40:34.092697192 +0000 UTC m=+0.461180109 container start c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T14:40:34.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 ceph-osd[172274]: -- 172.21.15.189:0/3385481439 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df18080d00 con 0x55df18053400 2024-09-13T14:40:34.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 bash[172221]: c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 2024-09-13T14:40:34.507 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 systemd[1]: Started Ceph osd.5 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:40:34.555 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:40:34.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:34.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 81631 81615 94.4348 59.25 0.00939257 0.0105841 2024-09-13T14:40:34.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:34.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:34.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:34.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:34.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:34.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:34.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:34 smithi189 ceph-mon[90992]: pgmap v906: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 52850/233688 objects degraded (22.616%); 9553/233688 objects misplaced (4.088%); 396 KiB/s, 1 keys/s, 10 objects/s recovering 2024-09-13T14:40:34.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/395836026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:34.896 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T14:40:34.634+0000 7f3b92946440 -1 Falling back to public interface 2024-09-13T14:40:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:34 smithi017 ceph-mon[103001]: pgmap v906: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 52850/233688 objects degraded (22.616%); 9553/233688 objects misplaced (4.088%); 396 KiB/s, 1 keys/s, 10 objects/s recovering 2024-09-13T14:40:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/395836026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:34 smithi017 ceph-mon[107556]: pgmap v906: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 52850/233688 objects degraded (22.616%); 9553/233688 objects misplaced (4.088%); 396 KiB/s, 1 keys/s, 10 objects/s recovering 2024-09-13T14:40:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/395836026' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:34.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:35.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:35.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:35.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:35.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:35.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:35.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:35.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:35.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:35.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:35.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 82531 82515 93.7406 56.25 0.0132274 0.0106616 2024-09-13T14:40:35.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:35.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:35.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:35.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:35.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:35.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:35.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:35.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:36.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:36.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:36.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:36.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:36.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:36.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:36.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:36.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:no valid command found; 10 closest matches: 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:0 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:1 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:2 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:abort 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:assert 2024-09-13T14:40:36.488 INFO:teuthology.orchestra.run.smithi189.stderr:config diff 2024-09-13T14:40:36.489 INFO:teuthology.orchestra.run.smithi189.stderr:config diff get 2024-09-13T14:40:36.489 INFO:teuthology.orchestra.run.smithi189.stderr:config get 2024-09-13T14:40:36.489 INFO:teuthology.orchestra.run.smithi189.stderr:config help [] 2024-09-13T14:40:36.489 INFO:teuthology.orchestra.run.smithi189.stderr:config set ... 2024-09-13T14:40:36.489 INFO:teuthology.orchestra.run.smithi189.stderr:admin_socket: invalid command 2024-09-13T14:40:36.495 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:36.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 83401 83385 93.0377 54.375 0.0239776 0.0107423 2024-09-13T14:40:36.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:36.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:36.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:36.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:36 smithi189 ceph-mon[90992]: pgmap v907: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 53718/240783 objects degraded (22.310%); 9438/240783 objects misplaced (3.920%); 1.2 MiB/s, 1 keys/s, 22 objects/s recovering 2024-09-13T14:40:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:36.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:36.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:36.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:36.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:36.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:36 smithi017 ceph-mon[103001]: pgmap v907: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 53718/240783 objects degraded (22.310%); 9438/240783 objects misplaced (3.920%); 1.2 MiB/s, 1 keys/s, 22 objects/s recovering 2024-09-13T14:40:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:36 smithi017 ceph-mon[107556]: pgmap v907: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 53718/240783 objects degraded (22.310%); 9438/240783 objects misplaced (3.920%); 1.2 MiB/s, 1 keys/s, 22 objects/s recovering 2024-09-13T14:40:36.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:37.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:37.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:37.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:40:37.261 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-09-13T14:40:37.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:37.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:37.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:37.425 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:37.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:37.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:37.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:37.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:37.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 84257 84241 92.3441 53.5 0.0102209 0.0108238 2024-09-13T14:40:37.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:37.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:37.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:37] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-09-13T14:40:37.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:37.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:37.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:37.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:37.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:38.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:38.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:38.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.237 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:38.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:38.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:38.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:38.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:38.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:38.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 85189 85173 91.7563 58.25 0.0277566 0.0108928 2024-09-13T14:40:38.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:40:38.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:38.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:38.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:38.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:38 smithi017 ceph-mon[103001]: pgmap v908: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 54288/244968 objects degraded (22.161%); 9692/244968 objects misplaced (3.956%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:40:38.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:38 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 54288/244968 objects degraded (22.161%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:38.802 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:38 smithi017 ceph-mon[107556]: pgmap v908: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 54288/244968 objects degraded (22.161%); 9692/244968 objects misplaced (3.956%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:40:38.802 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:38 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 54288/244968 objects degraded (22.161%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:38.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:38.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:38.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:38.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:38 smithi189 ceph-mon[90992]: pgmap v908: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 54288/244968 objects degraded (22.161%); 9692/244968 objects misplaced (3.956%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:40:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:38 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 54288/244968 objects degraded (22.161%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:39.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:39.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:39.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:39.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:39.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:39.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:39.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:39.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:39.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:39.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 86102 86086 91.1683 57.0625 0.0126873 0.0109625 2024-09-13T14:40:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:40:39.597 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:40:39.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:39.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:39.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:39.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:39.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:39.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:39.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:40.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:40.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:40.131 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:40.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:40.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:40.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:40.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:40.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:40:40.565424+0000 min lat: 0.00122954 max lat: 2.27384 avg lat: 0.0110304 2024-09-13T14:40:40.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:40:40.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 87031 87015 90.6165 58.0625 0.0195727 0.0110304 2024-09-13T14:40:40.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:40.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:40.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:40.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:40.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:40 smithi017 ceph-mon[103001]: pgmap v909: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 55131/250941 objects degraded (21.970%); 9581/250941 objects misplaced (3.818%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:40:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:40 smithi017 ceph-mon[107556]: pgmap v909: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 55131/250941 objects degraded (21.970%); 9581/250941 objects misplaced (3.818%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:40:40.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:40.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:40 smithi189 ceph-mon[90992]: pgmap v909: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 55131/250941 objects degraded (21.970%); 9581/250941 objects misplaced (3.818%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:40:41.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:41.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:41.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:41.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:41.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:41.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:41.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:41.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:41.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 87880 87864 90.0009 53.0625 0.00807384 0.0111044 2024-09-13T14:40:41.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:41.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:40:41.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:41.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:41.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:41.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:41.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:41.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:42.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:42.055 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:42.157 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:42.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:42.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:40:42.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:42.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:42.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:42.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:42.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 88702 88686 89.3779 51.375 0.00895482 0.0111691 2024-09-13T14:40:42.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:42.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:42.616 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:40:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:42.666 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T14:40:42.299+0000 7f3b92946440 -1 osd.5 294 log_to_monitors true 2024-09-13T14:40:42.718 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:42.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:42.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:42.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:42.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:42.930 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T14:40:42.728+0000 7f3b86eeb640 -1 osd.5 315 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:40:42.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-mon[90992]: pgmap v910: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 56256/257691 objects degraded (21.831%); 9581/257691 objects misplaced (3.718%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:40:42.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:42.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-mon[90992]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.931 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:42 smithi189 ceph-mon[90992]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[103001]: pgmap v910: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 56256/257691 objects degraded (21.831%); 9581/257691 objects misplaced (3.718%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[103001]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[103001]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[107556]: pgmap v910: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 56256/257691 objects degraded (21.831%); 9581/257691 objects misplaced (3.718%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[107556]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:42 smithi017 ceph-mon[107556]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T14:40:42.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:43.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:43.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:43.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:43.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:43.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:43.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:43.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:43.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:43.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:43.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:43.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 89556 89540 88.8064 53.375 0.00396206 0.0112547 2024-09-13T14:40:43.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:43.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:43.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:43.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:43.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:43.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:43.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[103001]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[103001]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[103001]: osdmap e315: 8 total, 7 up, 8 in 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[103001]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 56256/257691 objects degraded (21.831%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[107556]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:40:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[107556]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[107556]: osdmap e315: 8 total, 7 up, 8 in 2024-09-13T14:40:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[107556]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 56256/257691 objects degraded (21.831%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:43 smithi189 ceph-mon[90992]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:40:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:43 smithi189 ceph-mon[90992]: from='osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:43 smithi189 ceph-mon[90992]: osdmap e315: 8 total, 7 up, 8 in 2024-09-13T14:40:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:43 smithi189 ceph-mon[90992]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]} : dispatch 2024-09-13T14:40:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 56256/257691 objects degraded (21.831%), 18 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:44.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:44.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:44.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:44.192 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:44.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:44.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:44.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:44.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:44.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:44.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 89700 89684 87.5596 9 0.00357387 0.0112649 2024-09-13T14:40:44.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:44.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:44.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:44.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:44 smithi189 ceph-mon[90992]: pgmap v912: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 56801/260931 objects degraded (21.769%); 9835/260931 objects misplaced (3.769%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:40:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:44 smithi189 ceph-mon[90992]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:44.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:44 smithi189 ceph-mon[90992]: osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081] boot 2024-09-13T14:40:44.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:44 smithi189 ceph-mon[90992]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:40:44.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:40:44.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:44.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:44.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[103001]: pgmap v912: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 56801/260931 objects degraded (21.769%); 9835/260931 objects misplaced (3.769%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:40:44.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[103001]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:44.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[103001]: osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081] boot 2024-09-13T14:40:44.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[103001]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:40:44.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:40:44.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[107556]: pgmap v912: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 13 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s; 56801/260931 objects degraded (21.769%); 9835/260931 objects misplaced (3.769%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:40:44.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[107556]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:44.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[107556]: osd.5 [v2:172.21.15.189:6808/1980893081,v1:172.21.15.189:6809/1980893081] boot 2024-09-13T14:40:44.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[107556]: osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:40:44.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:40:44.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:44.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:44.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:45.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:45.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:45.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:45.168 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:45.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:45.205 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:40:45.206 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:40:45.206 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:40:45.206 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:40:45.268 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:45.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:45.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:45.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:45.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:45.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:45.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:45.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 89700 89684 86.2124 0 - 0.0112649 2024-09-13T14:40:45.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:45.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:45.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:45.697 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:40:45.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:45.790 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: pgmap v915: 99 pgs: 3 remapped+peering, 4 peering, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 10 active+undersized, 10 active+undersized+degraded, 64 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 40738/266670 objects degraded (15.277%); 5436/266670 objects misplaced (2.038%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3474298054' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[103001]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:40:45.791 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: pgmap v915: 99 pgs: 3 remapped+peering, 4 peering, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 10 active+undersized, 10 active+undersized+degraded, 64 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 40738/266670 objects degraded (15.277%); 5436/266670 objects misplaced (2.038%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3474298054' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:45.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:45 smithi017 ceph-mon[107556]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:40:45.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:45.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:45.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:45.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: pgmap v915: 99 pgs: 3 remapped+peering, 4 peering, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 10 active+undersized, 10 active+undersized+degraded, 64 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 40738/266670 objects degraded (15.277%); 5436/266670 objects misplaced (2.038%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3474298054' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:40:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:45 smithi189 ceph-mon[90992]: osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:40:46.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:46.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:46.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:46.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:46.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:40:46.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:46.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:46.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:46.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:46.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 89966 89950 85.1582 8.3125 0.00752522 0.0116694 2024-09-13T14:40:46.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:46.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:46.694 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:40:46.694 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:40:46.694 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:40:46.694 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:40:46.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:46.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:46.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:46.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:46.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:46.986 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:46.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:46.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:47.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:47.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:47.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:47.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:47.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:40:47.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:47.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:47.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:47.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:47.558 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:47] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:40:47.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 89986 89970 83.906 1.25 0.00763298 0.0116684 2024-09-13T14:40:47.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:47.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:47.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:47.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:47.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:47.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:47.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:47.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:47.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:48.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:48.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:48.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:48.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:48.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:48.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:48.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:48 smithi017 ceph-mon[103001]: pgmap v917: 99 pgs: 5 remapped+peering, 4 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 9 active+undersized, 6 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 35478/269052 objects degraded (13.186%); 2/269052 objects misplaced (0.001%); 1.6 MiB/s, 24 objects/s recovering 2024-09-13T14:40:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:48 smithi017 ceph-mon[107556]: pgmap v917: 99 pgs: 5 remapped+peering, 4 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 9 active+undersized, 6 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 35478/269052 objects degraded (13.186%); 2/269052 objects misplaced (0.001%); 1.6 MiB/s, 24 objects/s recovering 2024-09-13T14:40:48.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:48 smithi189 ceph-mon[90992]: pgmap v917: 99 pgs: 5 remapped+peering, 4 peering, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 9 active+undersized, 6 active+undersized+degraded, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 35478/269052 objects degraded (13.186%); 2/269052 objects misplaced (0.001%); 1.6 MiB/s, 24 objects/s recovering 2024-09-13T14:40:48.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:48.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:40:48.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 89986 89970 82.6723 0 - 0.0116684 2024-09-13T14:40:48.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:48.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:48.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:48.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:48.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:48.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:48.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:48.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:48.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:48.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:49.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:49.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:49.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:49.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:49.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:49.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:49.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:40:49.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:40:49.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:40:49.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:40:49.339 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:40:49.339 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:40:49.346 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:49.348 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:49.358 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:40:49.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:40:49.365 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:49.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:49.394 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:49.482 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35478/269052 objects degraded (13.186%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:49.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35478/269052 objects degraded (13.186%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:49.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35478/269052 objects degraded (13.186%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-09-13T14:40:49.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:49.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 89986 89970 81.4743 0 - 0.0116684 2024-09-13T14:40:49.612 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:49.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:49.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:49.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:49.859 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:49.882 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:49.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:49.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:50.008 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:50.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:50.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:50.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:50.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:50.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:50.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:50 smithi017 ceph-mon[103001]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 30763/269532 objects degraded (11.413%); 4416/269532 objects misplaced (1.638%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:40:50.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2682175930' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:50.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:50 smithi017 ceph-mon[107556]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 30763/269532 objects degraded (11.413%); 4416/269532 objects misplaced (1.638%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:40:50.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2682175930' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:50.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:50.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:50.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:50 smithi189 ceph-mon[90992]: pgmap v918: 99 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 30763/269532 objects degraded (11.413%); 4416/269532 objects misplaced (1.638%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:40:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2682175930' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:40:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:40:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:40:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:50.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:50.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:50.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 89986 89970 80.3105 0 - 0.0116684 2024-09-13T14:40:50.612 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:50.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:50.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:50.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:50.862 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:50.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:50.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:50.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:51.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:51.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:51.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:51.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:51.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:51.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:51.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:51.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:51.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:51.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 89986 89970 79.1794 0 - 0.0116684 2024-09-13T14:40:51.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:51.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:51.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:51.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:51.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:51.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:51.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:51.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:51.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:52.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:52.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:52.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:52.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:52.339 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:40:52.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:52.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:52.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:52.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:52 smithi189 ceph-mon[90992]: pgmap v919: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 53051/269994 objects degraded (19.649%); 4416/269994 objects misplaced (1.636%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:40:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3668514613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:52 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 89986 89970 78.0799 0 - 0.0116684 2024-09-13T14:40:52.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:52.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:52.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[103001]: pgmap v919: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 53051/269994 objects degraded (19.649%); 4416/269994 objects misplaced (1.636%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:40:52.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3668514613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[107556]: pgmap v919: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 53051/269994 objects degraded (19.649%); 4416/269994 objects misplaced (1.636%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:40:52.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3668514613' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:52 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:40:52.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:52.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:52.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:52.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:52.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:52.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:53.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:53.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:53.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:53.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:53.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:53.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:53.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:53 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:40:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:53 smithi017 ceph-mon[103001]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:40:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:53 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:40:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:53 smithi017 ceph-mon[107556]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:40:53.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:53.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:53.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:53 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:40:53.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:53 smithi189 ceph-mon[90992]: osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:40:53.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 89986 89970 77.0105 0 - 0.0116684 2024-09-13T14:40:53.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:40:53.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:53.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:53.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:53.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:53.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:53.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:53.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:53.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:54.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:54.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:54.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:54.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:54.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:54.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:54.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:54.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:54.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:54 smithi017 ceph-mon[103001]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 5.8 MiB/s, 91 objects/s recovering 2024-09-13T14:40:54.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 53051/269994 objects degraded (19.649%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:54.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:54 smithi017 ceph-mon[107556]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 5.8 MiB/s, 91 objects/s recovering 2024-09-13T14:40:54.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 53051/269994 objects degraded (19.649%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:54 smithi189 ceph-mon[90992]: pgmap v921: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 5.8 MiB/s, 91 objects/s recovering 2024-09-13T14:40:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 53051/269994 objects degraded (19.649%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:54.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:54.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:54.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 89986 89970 75.97 0 - 0.0116684 2024-09-13T14:40:54.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:54.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:54.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:54.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:54.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:54.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:55.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:55.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:55.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:55.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:55.161 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:55.227 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:40:55.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:55.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:55.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:55.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:55.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:55.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:55.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:40:55.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:55.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:55.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 89986 89970 74.9572 0 - 0.0116684 2024-09-13T14:40:55.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:55.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:55.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:55.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:55.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:55.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:55.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:56.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:56.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:56.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:56.153 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:56.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:56.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:56.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:56.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:56.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:56 smithi189 ceph-mon[90992]: pgmap v922: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 4.9 MiB/s, 77 objects/s recovering 2024-09-13T14:40:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:56 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:56.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:56 smithi189 ceph-mon[90992]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:40:56.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:56.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 89986 89970 73.971 0 - 0.0116684 2024-09-13T14:40:56.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:56.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[107556]: pgmap v922: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 4.9 MiB/s, 77 objects/s recovering 2024-09-13T14:40:56.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:56.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:56.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[107556]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:40:56.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[103001]: pgmap v922: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 85 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 3.2 MiB/s wr, 51 op/s; 53051/269994 objects degraded (19.649%); 3997/269994 objects misplaced (1.480%); 4.9 MiB/s, 77 objects/s recovering 2024-09-13T14:40:56.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:40:56.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3565052260' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:40:56.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:56 smithi017 ceph-mon[103001]: osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:40:56.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:56.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:56.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:56.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:56.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:56.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:56.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:56.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:57.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:57.119 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:40:57.119 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:40:57.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:57.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:57.149 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:40:57.149 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:40:57.165 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:40:57.165 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:40:57.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:40:57.182 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:40:57.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:57.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:57.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.394 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:57.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:57.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:57.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:57.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 89986 89970 73.0105 0 - 0.0116684 2024-09-13T14:40:57.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:57.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:57.684 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:40:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:40:57] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-13T14:40:57.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:57.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:57.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:57.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:57.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:57.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:58.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:58.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:58.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:58.140 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:58.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:58.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[103001]: pgmap v924: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s; 62942/269994 objects degraded (23.312%); 3997/269994 objects misplaced (1.480%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:58.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:58 smithi017 ceph-mon[107556]: pgmap v924: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s; 62942/269994 objects degraded (23.312%); 3997/269994 objects misplaced (1.480%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:40:58.369 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:58.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:58.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:40:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:40:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:58 smithi189 ceph-mon[90992]: pgmap v924: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s; 62942/269994 objects degraded (23.312%); 3997/269994 objects misplaced (1.480%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:40:58.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:40:58.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 89986 89970 72.0746 0 - 0.0116684 2024-09-13T14:40:58.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:40:58.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:40:58.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:40:58.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:58.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:58.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:40:58.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:58.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:40:58.901 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:40:58.901 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:40:58.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:59.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:59.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.127 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:40:59.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:40:59.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:40:59.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.390 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/304428759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62942/269994 objects degraded (23.312%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/304428759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:40:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62942/269994 objects degraded (23.312%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:59.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:40:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:59 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/304428759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:59 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:40:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:40:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62942/269994 objects degraded (23.312%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:59.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:40:59.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 89986 89970 71.1623 0 - 0.0116684 2024-09-13T14:40:59.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:40:59.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:40:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:40:59.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:40:59.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:40:59.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:40:59.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:40:59.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:40:59.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:40:59.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:00.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:00.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:41:00.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:00.137 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:00.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[103001]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:41:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[103001]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T14:41:00.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:00.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[107556]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:41:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:00 smithi017 ceph-mon[107556]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T14:41:00.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:00.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:00.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:00.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:41:00.363 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:00.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:00.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:00 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:41:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:00 smithi189 ceph-mon[90992]: osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:41:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:00 smithi189 ceph-mon[90992]: pgmap v926: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T14:41:00.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:00.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:00.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:41:00.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:00.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:00.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:41:00.567885+0000 min lat: 0.00122954 max lat: 2.47119 avg lat: 0.0116684 2024-09-13T14:41:00.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:41:00.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 89986 89970 70.2729 0 - 0.0116684 2024-09-13T14:41:00.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:00.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:00.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:00.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:00.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:00.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:00.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:00.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:00.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:00.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:01.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:01.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:01.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:01.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:01.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:01.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:01.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:01.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:01.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:01.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 89986 89970 69.4054 0 - 0.0116684 2024-09-13T14:41:01.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:01.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:01.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:01.761 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:01.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:01.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:01.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:01.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:02.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:02.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:02.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:02.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:02.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:02 smithi189 ceph-mon[90992]: pgmap v927: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:02.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:02.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:02.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:02.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:02.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:02.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:02 smithi017 ceph-mon[107556]: pgmap v927: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:02.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:02 smithi017 ceph-mon[103001]: pgmap v927: 99 pgs: 1 activating, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62942/269994 objects degraded (23.312%); 3563/269994 objects misplaced (1.320%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:02.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 90012 89996 68.5789 0.108333 0.0107691 0.0124194 2024-09-13T14:41:02.591 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:02.594 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:02.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:02.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:02.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:02.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:02.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:02.895 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:02.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:02.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:03.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:03.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:03.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:03.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:03.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:03.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:03.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:03.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:03.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:03.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:03.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:03.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 90191 90175 67.8875 11.1875 0.00931502 0.0142609 2024-09-13T14:41:03.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:03.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:03.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:03.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:03.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:03.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:03.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:04.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:04.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:04.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:04.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:04.214 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:41:04.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:04.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:04.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:04.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[103001]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 136 KiB/s wr, 2 op/s; 62942/270045 objects degraded (23.308%); 3100/270045 objects misplaced (1.148%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62942/270045 objects degraded (23.308%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3830028548' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[107556]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 136 KiB/s wr, 2 op/s; 62942/270045 objects degraded (23.308%); 3100/270045 objects misplaced (1.148%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62942/270045 objects degraded (23.308%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3830028548' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:04 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:04 smithi189 ceph-mon[90992]: pgmap v928: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 activating, 9 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 136 KiB/s wr, 2 op/s; 62942/270045 objects degraded (23.308%); 3100/270045 objects misplaced (1.148%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62942/270045 objects degraded (23.308%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3830028548' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:04 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:41:04.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:04.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:04.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 90372 90356 67.2141 11.3125 0.00650401 0.0147566 2024-09-13T14:41:04.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:04.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:04.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:04.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:04.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:04.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:04.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:05.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:05.039 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:05.141 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:05.166 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:05.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:41:05.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:41:05.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:41:05.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:05.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:05.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:05.243 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:41:05.243 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:05.263 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:41:05.263 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:05.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:05.283 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:41:05.283 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:05.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:05.305 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:41:05.305 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:05.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:41:05.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[103001]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3516726259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:05.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[107556]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:41:05.336 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3516726259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:05.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:05.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:05 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:05 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:41:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:05 smithi189 ceph-mon[90992]: osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:41:05.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3516726259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:05.521 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:05.544 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:05.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 90539 90523 66.5462 10.4375 0.00437085 0.0148927 2024-09-13T14:41:05.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:05.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:05.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:05.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:05.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:05.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:05.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:05.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:05.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:06.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:06.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:06.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:06.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:06.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:06.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:06.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:06.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:06.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:06 smithi017 ceph-mon[107556]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 62942/270534 objects degraded (23.266%); 3100/270534 objects misplaced (1.146%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:06 smithi017 ceph-mon[103001]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 62942/270534 objects degraded (23.266%); 3100/270534 objects misplaced (1.146%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:06.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:06 smithi189 ceph-mon[90992]: pgmap v930: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 62942/270534 objects degraded (23.266%); 3100/270534 objects misplaced (1.146%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:41:06.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:06.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 90772 90756 65.9418 14.5625 0.00632558 0.0150396 2024-09-13T14:41:06.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:06.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:06.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:06.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:06.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:06.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:06.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:06.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:06.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:07.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:07.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:07.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:07.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:07.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:07.347 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:07.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:07.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:07.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:07.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 90946 90930 65.3089 10.875 0.00755682 0.015185 2024-09-13T14:41:07.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:07.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:07.602 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:07] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-13T14:41:07.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:07.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:07.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:07.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:07.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:41:07.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:07.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:07.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:08.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:08.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:08.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:08.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:08.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:08.359 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:08.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:08.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:08 smithi017 ceph-mon[103001]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s; 62942/271296 objects degraded (23.200%); 3100/271296 objects misplaced (1.143%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:41:08.464 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:08 smithi017 ceph-mon[107556]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s; 62942/271296 objects degraded (23.200%); 3100/271296 objects misplaced (1.143%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:41:08.484 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:08.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:08 smithi189 ceph-mon[90992]: pgmap v931: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s; 62942/271296 objects degraded (23.200%); 3100/271296 objects misplaced (1.143%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:41:08.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:41:08.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 91104 91088 64.6791 9.875 0.00355445 0.01535 2024-09-13T14:41:08.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:08.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:08.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:08.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:08.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:08.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:08.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:08.908 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214850 6.19 6:9efaf998:::benchmark_data_smithi017_185083_object90238:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40373868199998, 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1610996929999999, 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.909 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214850 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:41:08.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487202+0000", 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6e-07 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487204+0000", 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2890000000000001e-06 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487207+0000", 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7160000000000001e-06 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487210+0000", 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0410000000000002e-06 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534594+0000", 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047383458000000003 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534603+0000", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4509999999999998e-06 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632356+0000", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0977522900000001 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632494+0000", 2024-09-13T14:41:08.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138255 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.632917+0000", 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042332999999999997 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640108+0000", 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071911279999999998 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640419+0000", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031022999999999999 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640470+0000", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1239999999999997e-05 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640785+0000", 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031557999999999998 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640827+0000", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1998e-05 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648079+0000", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007251955 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648150+0000", 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1000999999999993e-05 2024-09-13T14:41:08.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648302+0000", 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151471 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214865 6.19 6:990a3470:::benchmark_data_smithi017_185083_object90253:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40372514500001, 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.161541361, 2024-09-13T14:41:08.917 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214865 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487216+0000", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.03e-07 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487217+0000", 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1200000000000001e-06 2024-09-13T14:41:08.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487221+0000", 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9670000000000003e-06 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487224+0000", 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7829999999999999e-06 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534628+0000", 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.04740403 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534634+0000", 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636252+0000", 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.101618046 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636387+0000", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013459400000000001 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.636762+0000", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037534800000000001 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640114+0000", 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033514040000000001 2024-09-13T14:41:08.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640868+0000", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075420200000000004 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640935+0000", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7180000000000004e-05 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.640981+0000", 2024-09-13T14:41:08.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5516000000000003e-05 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641023+0000", 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2682000000000002e-05 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648552+0000", 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075286629999999997 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648628+0000", 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6207999999999999e-05 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648757+0000", 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128854 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214872 6.19 6:98033914:::benchmark_data_smithi017_185083_object90260:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:41:08.925 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.403678958, 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170118998, 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214872 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:41:08.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487262+0000", 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487264+0000", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479999999999998e-06 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487269+0000", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2870000000000004e-06 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487274+0000", 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8269999999999997e-06 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534661+0000", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047386510999999999 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534667+0000", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0959999999999997e-06 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641055+0000", 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1063884900000001 2024-09-13T14:41:08.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641187+0000", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013205500000000001 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.641618+0000", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000430772 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648355+0000", 2024-09-13T14:41:08.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006736952 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648368+0000", 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2619000000000001e-05 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648401+0000", 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3334000000000003e-05 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648438+0000", 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7083999999999999e-05 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648476+0000", 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7651000000000003e-05 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657179+0000", 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087027679999999996 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657239+0000", 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0705000000000002e-05 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657381+0000", 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014156800000000001 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214886 6.19 6:9efbdfd4:::benchmark_data_smithi017_185083_object90274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.933 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40365060599999, 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170313833, 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214886 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487290+0000", 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1e-07 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487292+0000", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7269999999999999e-06 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487296+0000", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7069999999999998e-06 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487299+0000", 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9490000000000001e-06 2024-09-13T14:41:08.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534693+0000", 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047393909999999997 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534699+0000", 2024-09-13T14:41:08.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0889999999999996e-06 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.644543+0000", 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1098440000000001 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.644779+0000", 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023615299999999999 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.645439+0000", 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00065987700000000001 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648512+0000", 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030732929999999999 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656850+0000", 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083375059999999997 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656918+0000", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8691999999999999e-05 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656972+0000", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3526999999999998e-05 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657014+0000", 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2472000000000003e-05 2024-09-13T14:41:08.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657426+0000", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00041171300000000002 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657478+0000", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2320000000000001e-05 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657604+0000", 2024-09-13T14:41:08.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012558799999999999 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214932 6.19 6:9ca5d594:::benchmark_data_smithi017_185083_object90320:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40363764699998, 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.170562039, 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.942 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214932 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487303+0000", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2100000000000001e-07 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487308+0000", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.634e-06 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487313+0000", 2024-09-13T14:41:08.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9479999999999999e-06 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487316+0000", 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8100000000000002e-06 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534724+0000", 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047408690000000003 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534732+0000", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7989999999999994e-06 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648787+0000", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1140544859999999 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.648928+0000", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141581 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.649304+0000", 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037605400000000001 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.656933+0000", 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076289690000000002 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657056+0000", 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012271999999999999 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657096+0000", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9823000000000001e-05 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657138+0000", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2122000000000001e-05 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657648+0000", 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051048300000000005 2024-09-13T14:41:08.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657690+0000", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2032e-05 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657740+0000", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9747999999999999e-05 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657865+0000", 2024-09-13T14:41:08.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124919 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214964 6.19 6:9cdc11f4:::benchmark_data_smithi017_185083_object90352:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40360585799999, 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1837455560000001, 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.950 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214964 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:41:08.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487335+0000", 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6899999999999999e-07 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487337+0000", 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3960000000000001e-06 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487341+0000", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.464e-06 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487344+0000", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7750000000000001e-06 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534759+0000", 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047414748999999999 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534765+0000", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6479999999999997e-06 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.651636+0000", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1168708469999999 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.651763+0000", 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126546 2024-09-13T14:41:08.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.654562+0000", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027995910000000001 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660645+0000", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060826470000000001 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660659+0000", 2024-09-13T14:41:08.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4129e-05 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667519+0000", 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068605159999999997 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667636+0000", 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116562 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667672+0000", 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5710000000000002e-05 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670895+0000", 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032235300000000001 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670948+0000", 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2732999999999998e-05 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671080+0000", 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013244400000000001 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:214989 6.19 6:9b173040:::benchmark_data_smithi017_185083_object90377:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40359270800002, 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1840168639999999, 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.958 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 214989 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487348+0000", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487350+0000", 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.72e-06 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487355+0000", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3539999999999999e-06 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487361+0000", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1379999999999998e-06 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534793+0000", 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047432329000000002 2024-09-13T14:41:08.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534799+0000", 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1720000000000004e-06 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.657906+0000", 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.123107012 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.658034+0000", 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127694 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.658392+0000", 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035811099999999999 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.666020+0000", 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076280389999999997 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667562+0000", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015422929999999999 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667707+0000", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014456199999999999 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667742+0000", 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4925999999999997e-05 2024-09-13T14:41:08.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671119+0000", 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033768280000000001 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671190+0000", 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1470999999999994e-05 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671245+0000", 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4509000000000003e-05 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671365+0000", 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120375 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:215071 6.19 6:9b4254d5:::benchmark_data_smithi017_185083_object90459:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.966 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40357340499997, 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.184973252, 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 215071 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487367+0000", 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487368+0000", 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4799999999999999e-07 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487370+0000", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479999999999998e-06 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487375+0000", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7939999999999999e-06 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487380+0000", 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8470000000000003e-06 2024-09-13T14:41:08.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534825+0000", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047444403000000003 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534831+0000", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2090000000000002e-06 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660686+0000", 2024-09-13T14:41:08.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1258555370000001 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.660810+0000", 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012359900000000001 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.661169+0000", 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035927600000000001 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667600+0000", 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006430247 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.667776+0000", 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000176603 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671155+0000", 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003378944 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.671400+0000", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024505499999999999 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672123+0000", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072296399999999999 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672161+0000", 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7564999999999997e-05 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672217+0000", 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5998999999999997e-05 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672341+0000", 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012391400000000001 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.974 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35349.0:215109 6.19 6:9a809a80:::benchmark_data_smithi017_185083_object90497:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 425.40355283000002, 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1883943699999999, 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35349", 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 215109 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487388+0000", 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.84e-07 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487390+0000", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1069999999999999e-06 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487397+0000", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7499999999999997e-06 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.487402+0000", 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0819999999999998e-06 2024-09-13T14:41:08.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.978 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534856+0000", 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047453433000000003 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:03.534862+0000", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663372+0000", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.128510023 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663490+0000", 2024-09-13T14:41:08.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118367 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,3,5,7", 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.663825+0000", 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000334111 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.670848+0000", 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070239309999999998 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672351+0000", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015024109999999999 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672421+0000", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0005000000000005e-05 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.672461+0000", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0558e-05 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675488+0000", 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030265019999999999 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675579+0000", 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1113999999999995e-05 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675643+0000", 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3744999999999994e-05 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:34:04.675782+0000", 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001395 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:60915 8.c 8:38d5f543:::benchmark_data_smithi017_253885_object60914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:13.827797+0000", 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 55.063144205999997, 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11422974299999999, 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.984 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60915 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827797+0000", 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827797+0000", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827797+0000", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9999999999999998e-07 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827818+0000", 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0559000000000001e-05 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827824+0000", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9029999999999998e-06 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.827829+0000", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.741e-06 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.831365+0000", 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035354760000000001 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.831480+0000", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011565099999999999 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.831604+0000", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012378899999999999 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.837158+0000", 2024-09-13T14:41:08.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055540900000000002 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.837594+0000", 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00043593000000000001 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.941780+0000", 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.104185947 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.941910+0000", 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013022700000000001 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:13.942026+0000", 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011603 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:82839 8.e 8:7f7e62b2:::benchmark_data_smithi017_253885_object82838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:35.878254+0000", 2024-09-13T14:41:08.990 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.012686817999999, 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.1134595, 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82839 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878254+0000", 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878254+0000", 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878255+0000", 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4600000000000003e-07 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878306+0000", 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0921999999999998e-05 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878319+0000", 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3008e-05 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.878326+0000", 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8120000000000002e-06 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881206+0000", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028803370000000002 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881351+0000", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144807 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881499+0000", 2024-09-13T14:41:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014786000000000001 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.988244+0000", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10674568800000001 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.988311+0000", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6554e-05 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.988363+0000", 2024-09-13T14:41:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1931000000000001e-05 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.988449+0000", 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5877999999999993e-05 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.991713+0000", 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032648569999999999 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:08.996 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:82842 8.e 8:78b86351:::benchmark_data_smithi017_253885_object82841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.008984194999996, 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.122549702, 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82842 2024-09-13T14:41:08.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:08.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881958+0000", 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.33e-06 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882008+0000", 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0303999999999998e-05 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882020+0000", 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1847e-05 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:08.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882027+0000", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0389999999999999e-06 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984746+0000", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10271846599999999 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984909+0000", 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016386699999999999 2024-09-13T14:41:09.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.985058+0000", 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000148479 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.991823+0000", 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067648980000000001 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996312+0000", 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044889270000000002 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996354+0000", 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1857999999999998e-05 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996415+0000", 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1398999999999999e-05 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.004506+0000", 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080912880000000003 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(8.2 314/311 [PushReplyOp(8:4757a975:::benchmark_data_smithi017_253885_object3344:head)])", 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.687871563999998, 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14042681400000001, 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.003 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203070+0000", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9399999999999993e-07 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203077+0000", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6320000000000002e-06 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203093+0000", 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6152000000000001e-05 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203102+0000", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999998e-06 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.334767+0000", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13166481199999999 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.343496+0000", 2024-09-13T14:41:09.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087293370000000002 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:83463 8.2 8:4b140280:::benchmark_data_smithi017_253885_object83462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:36.623318+0000", 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267622848000002, 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.120503367, 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.007 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83463 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623318+0000", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623318+0000", 2024-09-13T14:41:09.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623319+0000", 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.499e-06 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623343+0000", 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3587999999999998e-05 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623355+0000", 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1633e-05 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.623362+0000", 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2629999999999997e-06 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.716298+0000", 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.092936193 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.716471+0000", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017299899999999999 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.716636+0000", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016492899999999999 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.723582+0000", 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006946153 2024-09-13T14:41:09.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.723635+0000", 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2623000000000001e-05 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.740571+0000", 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016936395999999999 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.740651+0000", 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0290000000000005e-05 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.743821+0000", 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031698009999999999 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:84514 8.2 8:4b1745e2:::benchmark_data_smithi017_253885_object84513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:37.833844+0000", 2024-09-13T14:41:09.013 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 31.057096869999999, 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12164484, 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84514 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833844+0000", 2024-09-13T14:41:09.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833844+0000", 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833845+0000", 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.53e-06 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833871+0000", 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6041e-05 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833893+0000", 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1971999999999999e-05 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.833901+0000", 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.977e-06 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.854587+0000", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020685603 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.854739+0000", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015202299999999999 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.854910+0000", 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017058199999999999 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.870571+0000", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015660898999999999 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.870811+0000", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024058599999999999 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.952088+0000", 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081276721999999996 2024-09-13T14:41:09.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.952164+0000", 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6619000000000006e-05 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:37.955489+0000", 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033242860000000001 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:87664 8.2 8:4a551c3c:::benchmark_data_smithi017_253885_object87663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:41.264589+0000", 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 27.626351555999999, 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11438670099999999, 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 87664 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264589+0000", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264589+0000", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264590+0000", 2024-09-13T14:41:09.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4899999999999995e-07 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264638+0000", 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7744999999999998e-05 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264646+0000", 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3860000000000007e-06 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.264652+0000", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3029999999999996e-06 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.279749+0000", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015097124 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.279924+0000", 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000174289 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.324449+0000", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.044525694999999997 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.332203+0000", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077535110000000003 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.332226+0000", 2024-09-13T14:41:09.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2764000000000001e-05 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.374678+0000", 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042452102999999998 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.374978+0000", 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030038200000000003 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:41.378976+0000", 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039977499999999996 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88661 8.2 8:4a47af87:::benchmark_data_smithi017_253885_object88660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.454261+0000", 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.436680001999999, 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.120518444, 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88661 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454261+0000", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454261+0000", 2024-09-13T14:41:09.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454262+0000", 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4729999999999999e-06 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454290+0000", 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7409000000000001e-05 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454303+0000", 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3671e-05 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.454311+0000", 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1710000000000002e-06 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.476969+0000", 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022657515999999999 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477144+0000", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175487 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477324+0000", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017917000000000001 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571778+0000", 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.094454468 2024-09-13T14:41:09.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571797+0000", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8842e-05 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571850+0000", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3232000000000001e-05 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571927+0000", 2024-09-13T14:41:09.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6468e-05 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.574779+0000", 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002852537 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88690 8.e 8:774d5387:::benchmark_data_smithi017_253885_object88689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:41:09.032 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.413184640000001, 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13302814199999999, 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88690 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477757+0000", 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1900000000000001e-07 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477779+0000", 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2248000000000001e-05 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477791+0000", 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1515000000000001e-05 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477799+0000", 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8800000000000008e-06 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590385+0000", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112586266 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590547+0000", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162399 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590706+0000", 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015852800000000001 2024-09-13T14:41:09.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607019+0000", 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016312884999999999 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607063+0000", 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4583000000000003e-05 2024-09-13T14:41:09.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607112+0000", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8779000000000002e-05 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607171+0000", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8609000000000001e-05 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610784+0000", 2024-09-13T14:41:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003613631 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88698 8.e 8:7f48e2e9:::benchmark_data_smithi017_253885_object88697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.401544364999999, 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.125047359, 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.039 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88698 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:41:09.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489398+0000", 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5430000000000001e-06 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489430+0000", 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1897000000000003e-05 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489443+0000", 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2904999999999999e-05 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571286+0000", 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081843369999999999 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.594934+0000", 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023647714 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595081+0000", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014731100000000001 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595237+0000", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015634899999999999 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607076+0000", 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011838987 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610972+0000", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038959049999999999 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611023+0000", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0523999999999999e-05 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611085+0000", 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2390000000000004e-05 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.614444+0000", 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033584639999999998 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89686 8.2 8.4ba11582 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.167234781000001, 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97132002200000001, 2024-09-13T14:41:09.045 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89686 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723708+0000", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.531e-06 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723735+0000", 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.741e-05 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723752+0000", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7365000000000001e-05 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723760+0000", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1240000000000005e-06 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723831+0000", 2024-09-13T14:41:09.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0325000000000002e-05 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723842+0000", 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1249e-05 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695013+0000", 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97117079500000003 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:41:09.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695026+0000", 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3223e-05 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:41:09.050 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:41:09.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:09.173 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:09.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:09.224 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:41:09.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:09.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:09.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:09.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62942/271296 objects degraded (23.200%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1702373289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:09.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 91199 91183 64.0191 5.9375 0.00589353 0.0155048 2024-09-13T14:41:09.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:09.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:09.699 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62942/271296 objects degraded (23.200%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1702373289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62942/271296 objects degraded (23.200%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1702373289' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:41:09.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:09.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:09.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:09.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:09.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:10.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:10.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:10.173 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:10.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:10.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:10.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:10.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:41:10.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:10.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:10.479 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:10.481 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:10 smithi189 ceph-mon[90992]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s; 62942/272244 objects degraded (23.120%); 2663/272244 objects misplaced (0.978%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:41:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:10 smithi189 ceph-mon[90992]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:41:10.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:10.568 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 90 15 91338 91323 63.4051 8.75 0.0109406 0.0156492 2024-09-13T14:41:10.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[107556]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s; 62942/272244 objects degraded (23.120%); 2663/272244 objects misplaced (0.978%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[107556]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[103001]: pgmap v932: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s; 62942/272244 objects degraded (23.120%); 2663/272244 objects misplaced (0.978%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:41:10.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:10 smithi017 ceph-mon[103001]: osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:41:10.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:10.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:10.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:10.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:10.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:10.946 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:10.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:10.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:41:11.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:11.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:11.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:11.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:11.421 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:11.445 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:11.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 91.0083 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 91338 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 62.7264 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 51.5839 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 211.75 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:41:11.569 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1003 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 825.359 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3388 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0158507 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.230723 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 18.573 2024-09-13T14:41:11.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00122954 2024-09-13T14:41:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:11.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:11.579 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:41:11.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:11.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:11.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:11.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:11.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:11.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:11.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:12.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:12.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:12.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:12.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:12.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:12.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:12.342 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:12.367 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:12.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:12 smithi017 ceph-mon[103001]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s; 62942/273441 objects degraded (23.018%); 2663/273441 objects misplaced (0.974%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:12 smithi017 ceph-mon[107556]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s; 62942/273441 objects degraded (23.018%); 2663/273441 objects misplaced (0.974%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:12.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:12.495 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:12.495 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:12.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:12 smithi189 ceph-mon[90992]: pgmap v934: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s; 62942/273441 objects degraded (23.018%); 2663/273441 objects misplaced (0.974%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:41:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:12.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:12.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:12.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:12.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:12.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:12.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:12.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:12.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:12.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:13.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:13.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:13.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:13.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:13.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:13.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:13.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:13.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:13.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:13.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:13.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:13.689 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:41:13.689 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:13.707 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:41:13.708 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:13.725 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:41:13.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:13.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:13.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:41:13.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:13.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:13.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:13.765 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:41:13.765 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:13.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:41:13.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:13.804 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:41:13.805 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:13.825 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:41:13.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:13.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:13.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:13.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:13.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:14.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:14.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:14.129 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:14.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:14.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:14.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:14.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[103001]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 B/s rd, 8.5 MiB/s wr, 145 op/s; 62920/273636 objects degraded (22.994%); 2151/273636 objects misplaced (0.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:41:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62920/273636 objects degraded (22.994%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1549870205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[107556]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 B/s rd, 8.5 MiB/s wr, 145 op/s; 62920/273636 objects degraded (22.994%); 2151/273636 objects misplaced (0.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:41:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62920/273636 objects degraded (22.994%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1549870205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:14 smithi189 ceph-mon[90992]: pgmap v935: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 114 B/s rd, 8.5 MiB/s wr, 145 op/s; 62920/273636 objects degraded (22.994%); 2151/273636 objects misplaced (0.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:41:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62920/273636 objects degraded (22.994%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:14.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1549870205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:14.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:14.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:14.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:14.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:14.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:14.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:14.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:14.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:14.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:15.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:15.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:15.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:15.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:15.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:15.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:15.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:15.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:15.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:15.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:15.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:15.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:15.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:15.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:15.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:15.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:15.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:16.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:16.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:16.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:16.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:16.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:16.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:16.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:16.431 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:16.440 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:16 smithi017 ceph-mon[107556]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 159 op/s; 62920/272871 objects degraded (23.059%); 2151/272871 objects misplaced (0.788%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:41:16.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:16 smithi017 ceph-mon[103001]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 159 op/s; 62920/272871 objects degraded (23.059%); 2151/272871 objects misplaced (0.788%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:41:16.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:16 smithi189 ceph-mon[90992]: pgmap v936: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 159 op/s; 62920/272871 objects degraded (23.059%); 2151/272871 objects misplaced (0.788%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:41:16.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:16.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:16.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:16.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:16.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:16.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:16.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:16.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:16.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:16.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:16.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:17.038 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:17.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:17.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:17.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:17.232 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:41:17.249 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:17.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:17.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:17.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:17.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:17.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:17.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:41:17.540 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:17.562 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:17.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:17.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:17] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:41:17.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:17.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:17.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:17.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:17.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:18.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:18.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:18.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:18.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:18.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:18.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[103001]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 165 op/s; 62804/271953 objects degraded (23.094%); 2151/271953 objects misplaced (0.791%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[103001]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[107556]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 165 op/s; 62804/271953 objects degraded (23.094%); 2151/271953 objects misplaced (0.791%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:41:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:18.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:18.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:18 smithi017 ceph-mon[107556]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:41:18.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:18.445 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:18 smithi189 ceph-mon[90992]: pgmap v937: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 165 op/s; 62804/271953 objects degraded (23.094%); 2151/271953 objects misplaced (0.791%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:41:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:18 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:41:18.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3727620133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:41:18.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:18 smithi189 ceph-mon[90992]: osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:41:18.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:18.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:18.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:18.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:18.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:18.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:18.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:18.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:18.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:19.006 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:41:19.006 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:41:19.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:41:19.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:19.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:19.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:19.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:19.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:19.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:19.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62804/271953 objects degraded (23.094%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:19.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62804/271953 objects degraded (23.094%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:19.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:19.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:19.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62804/271953 objects degraded (23.094%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:19.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:19.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:19.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:19.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:19.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:19.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:19.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:19.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:19.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:20.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:20.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:20.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:41:20.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:20.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:20.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:20.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:20 smithi017 ceph-mon[103001]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 103 B/s rd, 3.9 MiB/s wr, 165 op/s; 62804/271041 objects degraded (23.171%); 1549/271041 objects misplaced (0.572%); 7.4 MiB/s, 119 objects/s recovering 2024-09-13T14:41:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:20 smithi017 ceph-mon[107556]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 103 B/s rd, 3.9 MiB/s wr, 165 op/s; 62804/271041 objects degraded (23.171%); 1549/271041 objects misplaced (0.572%); 7.4 MiB/s, 119 objects/s recovering 2024-09-13T14:41:20.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:20.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:20 smithi189 ceph-mon[90992]: pgmap v939: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 103 B/s rd, 3.9 MiB/s wr, 165 op/s; 62804/271041 objects degraded (23.171%); 1549/271041 objects misplaced (0.572%); 7.4 MiB/s, 119 objects/s recovering 2024-09-13T14:41:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:20.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:20.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:20.706 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:20.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:20.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:20.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:20.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:20.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:20.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:21.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:41:21.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:21.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:21.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:21.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:21.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:21.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:21.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:21.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:21.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:21.729 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:21.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:21.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:21.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:21.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:21.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:22.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:22.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:22.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:22.239 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:41:22.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:22.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:22.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[103001]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 169 op/s; 62804/269691 objects degraded (23.287%); 1549/269691 objects misplaced (0.574%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:41:22.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3253352118' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[107556]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 169 op/s; 62804/269691 objects degraded (23.287%); 1549/269691 objects misplaced (0.574%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:41:22.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3253352118' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:22.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:22.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:22 smithi189 ceph-mon[90992]: pgmap v940: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 169 op/s; 62804/269691 objects degraded (23.287%); 1549/269691 objects misplaced (0.574%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:41:22.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3253352118' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:41:22.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:22.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:22.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:22.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:22.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:22.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:22.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:22.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:22.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:23.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:23.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:23.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:23.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:23.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:23.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:23.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:23.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:23.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:23.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:23 smithi017 ceph-mon[107556]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:41:23.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:23.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:23 smithi017 ceph-mon[103001]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:41:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:23.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:41:23.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:23 smithi189 ceph-mon[90992]: osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:41:23.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:23.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:23.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:23.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:23.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:23.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:23.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:23.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:41:23.894 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:23.900 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:23.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:23.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:23.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:24.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:24.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:24.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:24.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:24.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:24.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:24.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:24 smithi017 ceph-mon[103001]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 62702/268830 objects degraded (23.324%); 778/268830 objects misplaced (0.289%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:41:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62702/268830 objects degraded (23.324%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:24 smithi017 ceph-mon[107556]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 62702/268830 objects degraded (23.324%); 778/268830 objects misplaced (0.289%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:41:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62702/268830 objects degraded (23.324%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:24.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:24 smithi189 ceph-mon[90992]: pgmap v942: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 62702/268830 objects degraded (23.324%); 778/268830 objects misplaced (0.289%); 12 MiB/s, 185 objects/s recovering 2024-09-13T14:41:24.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62702/268830 objects degraded (23.324%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:24.558 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:24.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:24.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:24.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:24.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:24.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:24.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:24.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:24.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:25.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:25.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:25.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:41:25.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:41:25.128 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:41:25.128 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:41:25.133 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:41:25.133 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:41:25.150 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:41:25.150 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:41:25.167 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:41:25.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:41:25.184 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:41:25.185 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:41:25.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:25.201 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:41:25.202 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:41:25.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:25.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:41:25.219 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:41:25.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:25.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:25.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:25.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1771138847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:25.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1771138847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:25.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:25.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1771138847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:41:25.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:25.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:25.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:25.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:25.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:25.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:25.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:25.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:25.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:26.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:26.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:26.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:26.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:26.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:26.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:26.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:26.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:26.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:26 smithi189 ceph-mon[90992]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 62702/267606 objects degraded (23.431%); 778/267606 objects misplaced (0.291%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:26.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:26.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:26 smithi017 ceph-mon[103001]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 62702/267606 objects degraded (23.431%); 778/267606 objects misplaced (0.291%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:26 smithi017 ceph-mon[107556]: pgmap v943: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 62702/267606 objects degraded (23.431%); 778/267606 objects misplaced (0.291%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:26.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:26.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:26.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:26.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:26.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:26.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:26.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:26.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:27.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:27.115 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:27.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:27.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:27.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:27.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:27.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:27.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:27.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:27.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:27.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:27] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-13T14:41:27.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:27.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:27.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:27.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:27.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:27.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:27.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:28.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:28.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:28.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:28.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:28.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:28.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[103001]: pgmap v944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/266730 objects degraded (23.449%); 778/266730 objects misplaced (0.292%); 9.6 MiB/s, 153 objects/s recovering 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[103001]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[107556]: pgmap v944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/266730 objects degraded (23.449%); 778/266730 objects misplaced (0.292%); 9.6 MiB/s, 153 objects/s recovering 2024-09-13T14:41:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:28 smithi017 ceph-mon[107556]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:41:28.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:28.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:28 smithi189 ceph-mon[90992]: pgmap v944: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/266730 objects degraded (23.449%); 778/266730 objects misplaced (0.292%); 9.6 MiB/s, 153 objects/s recovering 2024-09-13T14:41:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:28 smithi189 ceph-mon[90992]: osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:41:28.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:28.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:28.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:28.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:28.862 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:28.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:28.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:28.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:29.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:29.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:29.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:41:29.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:29.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:29.246 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:29.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:29.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:29.394 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:29 smithi017 ceph-mon[103001]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:41:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 62545/266730 objects degraded (23.449%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:29 smithi017 ceph-mon[107556]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:41:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 62545/266730 objects degraded (23.449%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:29.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:29.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:29.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:29.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:29.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:29.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:29 smithi189 ceph-mon[90992]: osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:41:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 62545/266730 objects degraded (23.449%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:41:29.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:29.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:29.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:29.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:29.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:30.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:30.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:30.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:30.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:30.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:30.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:30.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:30 smithi189 ceph-mon[90992]: pgmap v947: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/265722 objects degraded (23.538%); 15 MiB/s, 237 objects/s recovering 2024-09-13T14:41:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:30.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:30.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:30.643 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:30.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:30 smithi017 ceph-mon[103001]: pgmap v947: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/265722 objects degraded (23.538%); 15 MiB/s, 237 objects/s recovering 2024-09-13T14:41:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:30 smithi017 ceph-mon[107556]: pgmap v947: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 62545/265722 objects degraded (23.538%); 15 MiB/s, 237 objects/s recovering 2024-09-13T14:41:30.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:30.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:30.905 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:30.928 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:30.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:31.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:31.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:31.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:31.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:31.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:31.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:31.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:31.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:31.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:31.540 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:31.566 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:31.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:31.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:31.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:31.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:31.917 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:31.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:31.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:32.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:32.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:32.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:32.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:32.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:32.300 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:32.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:32.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:32.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:32 smithi189 ceph-mon[90992]: pgmap v948: 99 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 183 op/s; 57951/264435 objects degraded (21.915%); 4326/264435 objects misplaced (1.636%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:41:32.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:32.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:32.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:32 smithi017 ceph-mon[103001]: pgmap v948: 99 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 183 op/s; 57951/264435 objects degraded (21.915%); 4326/264435 objects misplaced (1.636%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:41:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:32 smithi017 ceph-mon[107556]: pgmap v948: 99 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 183 op/s; 57951/264435 objects degraded (21.915%); 4326/264435 objects misplaced (1.636%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:41:32.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.701 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:32.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:32.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:32.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:32.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:32.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:33.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:33.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:33.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:33.307 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:33.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:33.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:33.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:33.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:33.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:33.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:33.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:33.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:33.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:33.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:33.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:33.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:33.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:34.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:34.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:34.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:34.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:34.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:34.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:34.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:34.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:34.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:34.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:34.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:34 smithi017 ceph-mon[103001]: pgmap v949: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 57526/263502 objects degraded (21.831%); 4326/263502 objects misplaced (1.642%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:41:34.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 57526/263502 objects degraded (21.831%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:34.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:34 smithi017 ceph-mon[107556]: pgmap v949: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 57526/263502 objects degraded (21.831%); 4326/263502 objects misplaced (1.642%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:41:34.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 57526/263502 objects degraded (21.831%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:34.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:34.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:34.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:34.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:34 smithi189 ceph-mon[90992]: pgmap v949: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 57526/263502 objects degraded (21.831%); 4326/263502 objects misplaced (1.642%); 12 MiB/s, 189 objects/s recovering 2024-09-13T14:41:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 57526/263502 objects degraded (21.831%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:34.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:34.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:34.968 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:34.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:34.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:35.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:35.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:35.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:35.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:35.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:35.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:35.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:35.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:35.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:35.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:35.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:35.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:35.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:35.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:35.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:35.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:35.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:36.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:36.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:36.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:36.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:36.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:36.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:36.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:36.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:36.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:36.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:36.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:36 smithi017 ceph-mon[107556]: pgmap v950: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57526/262419 objects degraded (21.921%); 4326/262419 objects misplaced (1.649%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:36 smithi017 ceph-mon[103001]: pgmap v950: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57526/262419 objects degraded (21.921%); 4326/262419 objects misplaced (1.649%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:36.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:36 smithi189 ceph-mon[90992]: pgmap v950: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57526/262419 objects degraded (21.921%); 4326/262419 objects misplaced (1.649%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:41:36.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:36.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:36.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:36.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:36.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:37.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:37.200 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:37.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:37.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:37.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:37.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:37.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:37.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:37.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:37] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-13T14:41:37.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:37.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:37.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:37.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:37.957 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:37.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:37.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:38.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:38.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:38.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:38.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:38.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:38.360 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:38.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:38.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:38.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:38.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:38 smithi017 ceph-mon[103001]: pgmap v951: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57072/261399 objects degraded (21.833%); 3988/261399 objects misplaced (1.526%); 14 MiB/s, 216 objects/s recovering 2024-09-13T14:41:38.610 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:38 smithi017 ceph-mon[107556]: pgmap v951: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57072/261399 objects degraded (21.833%); 3988/261399 objects misplaced (1.526%); 14 MiB/s, 216 objects/s recovering 2024-09-13T14:41:38.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:38.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:38 smithi189 ceph-mon[90992]: pgmap v951: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 57072/261399 objects degraded (21.833%); 3988/261399 objects misplaced (1.526%); 14 MiB/s, 216 objects/s recovering 2024-09-13T14:41:38.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:38.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:38.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:38.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:38.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:39.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:39.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:39.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:39.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:39.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:39.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:39.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:39.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:39.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:39.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:39.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:39.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 57072/261399 objects degraded (21.833%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:39.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 57072/261399 objects degraded (21.833%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:39.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:39.770 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 57072/261399 objects degraded (21.833%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:39.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:39.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:39.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:39.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:40.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:40.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:40.127 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:41:40.128 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:41:40.128 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:41:40.128 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:41:40.128 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:41:40.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:40.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:40.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:40.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:40.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:40.483 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:40.486 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:40.490 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:41:40.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:40.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:40.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:40.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:40.736 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:40 smithi017 ceph-mon[103001]: pgmap v952: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 57072/260196 objects degraded (21.934%); 3988/260196 objects misplaced (1.533%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:41:40.736 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:40.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:40 smithi017 ceph-mon[107556]: pgmap v952: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 57072/260196 objects degraded (21.934%); 3988/260196 objects misplaced (1.533%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:41:40.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:40 smithi189 ceph-mon[90992]: pgmap v952: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 57072/260196 objects degraded (21.934%); 3988/260196 objects misplaced (1.533%); 7.8 MiB/s, 125 objects/s recovering 2024-09-13T14:41:40.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:41:40.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:40.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:40.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:40.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:40.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:41.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:41.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:41.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:41.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:41.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:41.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:41.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:41.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:41.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:41.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:41.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:41.698 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:41 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:41.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:41 smithi189 ceph-mon[90992]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:41:41.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:41.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:41.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:41.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:41.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[103001]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2237369999' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:41:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:41 smithi017 ceph-mon[107556]: osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:41:42.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:42.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:42.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:42.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:42.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:42.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:42.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:42.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:42.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:42.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[103001]: pgmap v954: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 57072/258606 objects degraded (22.069%); 3657/258606 objects misplaced (1.414%); 8.9 MiB/s, 143 objects/s recovering 2024-09-13T14:41:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[107556]: pgmap v954: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 57072/258606 objects degraded (22.069%); 3657/258606 objects misplaced (1.414%); 8.9 MiB/s, 143 objects/s recovering 2024-09-13T14:41:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:42.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:42.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:42 smithi189 ceph-mon[90992]: pgmap v954: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 57072/258606 objects degraded (22.069%); 3657/258606 objects misplaced (1.414%); 8.9 MiB/s, 143 objects/s recovering 2024-09-13T14:41:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:41:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:42.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:41:42.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:42.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:42.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:42.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:42.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:43.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:43.125 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:43.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:43.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:43.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:43.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:43.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:43.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:43.495 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:41:43.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:43.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:43.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:43.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:41:43.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:43.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:43.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:43.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:41:43.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:41:43.917 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:43.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:44.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:44.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:44.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:41:44.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:44.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:44.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:44.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:44.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:44.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:44.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:44.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:44.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:44.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:44.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:44 smithi189 ceph-mon[90992]: pgmap v955: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 56596/257688 objects degraded (21.963%); 3657/257688 objects misplaced (1.419%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:41:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:41:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:44 smithi189 ceph-mon[90992]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:41:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 56596/257688 objects degraded (21.963%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:44.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:44.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:44.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[107556]: pgmap v955: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 56596/257688 objects degraded (21.963%); 3657/257688 objects misplaced (1.419%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[107556]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 56596/257688 objects degraded (21.963%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[103001]: pgmap v955: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 56596/257688 objects degraded (21.963%); 3657/257688 objects misplaced (1.419%); 9.2 MiB/s, 147 objects/s recovering 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2946681223' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:41:44.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[103001]: osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:41:44.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 56596/257688 objects degraded (21.963%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:44.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:41:44.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:44.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:44.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:45.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:45.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:45.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:45.174 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:45.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:45.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:45.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:45.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:45.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:45.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:45.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:45.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:45.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:45.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:45.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:45.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:45.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:45.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:46.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:46.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:46.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:46.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:46.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:46.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:46.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:46.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:46.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:46.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:46.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:46.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:46.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:46 smithi189 ceph-mon[90992]: pgmap v957: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 56596/256554 objects degraded (22.060%); 3657/256554 objects misplaced (1.425%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:41:46.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:46.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:46.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:46.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:46.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:46.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:46 smithi017 ceph-mon[103001]: pgmap v957: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 56596/256554 objects degraded (22.060%); 3657/256554 objects misplaced (1.425%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:41:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:46 smithi017 ceph-mon[107556]: pgmap v957: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 56596/256554 objects degraded (22.060%); 3657/256554 objects misplaced (1.425%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:41:47.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:47.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:47.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:47.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:47.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:47.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:47.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:47.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:47.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:47.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:47.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:47.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:47] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:41:47.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:47.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:47.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:47.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:47.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:48.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:48.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:48.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:48.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:48.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:48.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:48.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:48.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:48.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:48.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:48.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:48.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:48.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:48.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:48.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:48.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:48.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:48.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:48 smithi017 ceph-mon[103001]: pgmap v958: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 56118/255837 objects degraded (21.935%); 3275/255837 objects misplaced (1.280%); 12 MiB/s, 195 objects/s recovering 2024-09-13T14:41:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:48 smithi017 ceph-mon[107556]: pgmap v958: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 56118/255837 objects degraded (21.935%); 3275/255837 objects misplaced (1.280%); 12 MiB/s, 195 objects/s recovering 2024-09-13T14:41:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:48 smithi189 ceph-mon[90992]: pgmap v958: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 56118/255837 objects degraded (21.935%); 3275/255837 objects misplaced (1.280%); 12 MiB/s, 195 objects/s recovering 2024-09-13T14:41:49.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.057 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:49.182 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:49.284 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:49.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:49.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:49.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.561 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:49.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:49.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:49.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:49.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:49.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:49.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:49.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:49.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 56118/255837 objects degraded (21.935%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 56118/255837 objects degraded (21.935%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 56118/255837 objects degraded (21.935%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:50.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:50.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:50.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:50.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:50.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:50.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:50.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:50.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:50.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:50.488 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:41:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:41:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:50.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:50.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:50.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:50.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:50.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:50.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:50.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:50 smithi017 ceph-mon[103001]: pgmap v959: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 56065/255327 objects degraded (21.958%); 3275/255327 objects misplaced (1.283%); 11 MiB/s, 182 objects/s recovering 2024-09-13T14:41:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:50 smithi017 ceph-mon[107556]: pgmap v959: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 56065/255327 objects degraded (21.958%); 3275/255327 objects misplaced (1.283%); 11 MiB/s, 182 objects/s recovering 2024-09-13T14:41:50.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:50.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:50.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:50 smithi189 ceph-mon[90992]: pgmap v959: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 56065/255327 objects degraded (21.958%); 3275/255327 objects misplaced (1.283%); 11 MiB/s, 182 objects/s recovering 2024-09-13T14:41:51.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:51.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:51.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:51.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:51.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:51.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:51.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:51.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:51.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:51.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:51.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:51.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:51.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:51.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:51.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:51.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:51.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:52.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:52.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:52.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:41:52.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:41:52.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:52.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:52.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:52.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:52.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:52.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:52.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:52.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:52.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:52.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:52 smithi017 ceph-mon[103001]: pgmap v960: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 56065/254769 objects degraded (22.006%); 2812/254769 objects misplaced (1.104%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:41:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:52 smithi017 ceph-mon[107556]: pgmap v960: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 56065/254769 objects degraded (22.006%); 2812/254769 objects misplaced (1.104%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:41:52.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:52.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:52.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:52 smithi189 ceph-mon[90992]: pgmap v960: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 56065/254769 objects degraded (22.006%); 2812/254769 objects misplaced (1.104%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:41:53.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:53.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:53.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:53.209 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:53.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:53.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:53.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:53.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:53.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:53.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:53.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:53.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:53.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:53.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:53.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:53.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:53.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:53.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:54.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:54.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:54.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:54.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:54.241 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:54.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:54.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:54.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:54.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:54.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:54.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:54.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:54 smithi017 ceph-mon[103001]: pgmap v961: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 55505/254310 objects degraded (21.826%); 2812/254310 objects misplaced (1.106%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:41:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 55505/254310 objects degraded (21.826%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:54 smithi017 ceph-mon[107556]: pgmap v961: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 55505/254310 objects degraded (21.826%); 2812/254310 objects misplaced (1.106%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:41:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 55505/254310 objects degraded (21.826%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:54.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:54.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:54.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:54.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:54.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:54.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:54 smithi189 ceph-mon[90992]: pgmap v961: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 55505/254310 objects degraded (21.826%); 2812/254310 objects misplaced (1.106%); 11 MiB/s, 180 objects/s recovering 2024-09-13T14:41:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 55505/254310 objects degraded (21.826%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:55.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:55.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:55.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:55.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:55.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:55.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:55.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:55.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:55.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:55.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:55.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:55.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:55.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:55.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:55.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:55.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:56.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:56.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:56.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:56.249 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:56.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:56.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:56.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:56.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:56.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.521 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:56.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:56.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:56.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:56 smithi017 ceph-mon[103001]: pgmap v962: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 55505/253548 objects degraded (21.891%); 2812/253548 objects misplaced (1.109%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:41:56.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:56 smithi017 ceph-mon[107556]: pgmap v962: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 55505/253548 objects degraded (21.891%); 2812/253548 objects misplaced (1.109%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:41:56.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:56.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:56.772 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:56.874 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:56.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:56.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:57.001 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:56 smithi189 ceph-mon[90992]: pgmap v962: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 55505/253548 objects degraded (21.891%); 2812/253548 objects misplaced (1.109%); 9.7 MiB/s, 154 objects/s recovering 2024-09-13T14:41:57.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:57.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:57.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:57.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:57.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:57.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:57.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:57.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:57.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:57.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:41:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:41:57] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:41:57.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:57.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:57.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:57.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:57.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:57.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:58.016 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:58.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:58.144 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:58.166 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:58.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:58.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:58.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:58.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:58.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:58.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:58.654 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:58.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:58.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:58.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:58.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:58.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:58.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:58 smithi017 ceph-mon[103001]: pgmap v963: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54983/252861 objects degraded (21.744%); 2318/252861 objects misplaced (0.917%); 15 MiB/s, 232 objects/s recovering 2024-09-13T14:41:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:58 smithi017 ceph-mon[107556]: pgmap v963: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54983/252861 objects degraded (21.744%); 2318/252861 objects misplaced (0.917%); 15 MiB/s, 232 objects/s recovering 2024-09-13T14:41:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:58 smithi189 ceph-mon[90992]: pgmap v963: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54983/252861 objects degraded (21.744%); 2318/252861 objects misplaced (0.917%); 15 MiB/s, 232 objects/s recovering 2024-09-13T14:41:59.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:59.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:59.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:41:59.182 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:59.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:41:59.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:59.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:59.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:59.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:59.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:59.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:59.560 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:59.661 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:59.684 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:59.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:59.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:59.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:41:59.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:59.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:41:59.936 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:41:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:41:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:41:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 54983/252861 objects degraded (21.744%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:41:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:41:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 54983/252861 objects degraded (21.744%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:41:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 54983/252861 objects degraded (21.744%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:00.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:00.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:00.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:00.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:00.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:00.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:00.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:00.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:00.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:00.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:00.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:00 smithi017 ceph-mon[103001]: pgmap v964: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 54921/252222 objects degraded (21.775%); 2318/252222 objects misplaced (0.919%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:42:00.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:00 smithi017 ceph-mon[107556]: pgmap v964: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 54921/252222 objects degraded (21.775%); 2318/252222 objects misplaced (0.919%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:42:00.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:00 smithi189 ceph-mon[90992]: pgmap v964: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 54921/252222 objects degraded (21.775%); 2318/252222 objects misplaced (0.919%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:42:00.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:00.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:01.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:01.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:01.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:01.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:01.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.462 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:01.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:01.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:01.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:01.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:02.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:02.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:02.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:02.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:02.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:02.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:02.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:02.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:02.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:02.599 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:02 smithi017 ceph-mon[103001]: pgmap v965: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 54921/251373 objects degraded (21.848%); 1786/251373 objects misplaced (0.710%); 13 MiB/s, 209 objects/s recovering 2024-09-13T14:42:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:02 smithi017 ceph-mon[107556]: pgmap v965: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 54921/251373 objects degraded (21.848%); 1786/251373 objects misplaced (0.710%); 13 MiB/s, 209 objects/s recovering 2024-09-13T14:42:02.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:02.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:02.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:02.975 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:02 smithi189 ceph-mon[90992]: pgmap v965: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 54921/251373 objects degraded (21.848%); 1786/251373 objects misplaced (0.710%); 13 MiB/s, 209 objects/s recovering 2024-09-13T14:42:03.077 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:03.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:03.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:03.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:03.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:03.484 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:03.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:03.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:03.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:03.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:03.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.961 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:04.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:04.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:04.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:04.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:04.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:04.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:04.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:04.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:04.744 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:04.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:04.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:04 smithi017 ceph-mon[103001]: pgmap v966: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 54275/250872 objects degraded (21.635%); 1786/250872 objects misplaced (0.712%); 14 MiB/s, 219 objects/s recovering 2024-09-13T14:42:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 54275/250872 objects degraded (21.635%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:04 smithi017 ceph-mon[107556]: pgmap v966: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 54275/250872 objects degraded (21.635%); 1786/250872 objects misplaced (0.712%); 14 MiB/s, 219 objects/s recovering 2024-09-13T14:42:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 54275/250872 objects degraded (21.635%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:04.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:04 smithi189 ceph-mon[90992]: pgmap v966: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 54275/250872 objects degraded (21.635%); 1786/250872 objects misplaced (0.712%); 14 MiB/s, 219 objects/s recovering 2024-09-13T14:42:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 54275/250872 objects degraded (21.635%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:05.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:05.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:05.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:05.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:05.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:05.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:05.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:05.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:05.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:05.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:05.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:05.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:05.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:06.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:06.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:06.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:06.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:06.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:06.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:06.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:06.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:06.707 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:06 smithi017 ceph-mon[103001]: pgmap v967: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 54275/250080 objects degraded (21.703%); 1786/250080 objects misplaced (0.714%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:42:06.707 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:06 smithi017 ceph-mon[107556]: pgmap v967: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 54275/250080 objects degraded (21.703%); 1786/250080 objects misplaced (0.714%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:42:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:06.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:06.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:06.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:06.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:06.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:06 smithi189 ceph-mon[90992]: pgmap v967: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 54275/250080 objects degraded (21.703%); 1786/250080 objects misplaced (0.714%); 11 MiB/s, 177 objects/s recovering 2024-09-13T14:42:07.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:07.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:07.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:07.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:07.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:07.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:07.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:07.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:07.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:07.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:07.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:07] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:42:07.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:07.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:07.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:07.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:07.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:07.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:08.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:08.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:08.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:08.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.372 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:08.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:08.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:08.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:08.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:08.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:08.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:08.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:08.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:08.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:08 smithi017 ceph-mon[103001]: pgmap v968: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 53568/249444 objects degraded (21.475%); 1103/249444 objects misplaced (0.442%); 18 MiB/s, 289 objects/s recovering 2024-09-13T14:42:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:08 smithi017 ceph-mon[107556]: pgmap v968: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 53568/249444 objects degraded (21.475%); 1103/249444 objects misplaced (0.442%); 18 MiB/s, 289 objects/s recovering 2024-09-13T14:42:08.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:08 smithi189 ceph-mon[90992]: pgmap v968: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 53568/249444 objects degraded (21.475%); 1103/249444 objects misplaced (0.442%); 18 MiB/s, 289 objects/s recovering 2024-09-13T14:42:09.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:09.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:09.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:09.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:09.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:09.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:09.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:09.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:09.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:09.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:09.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:09.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 53568/249444 objects degraded (21.475%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 53568/249444 objects degraded (21.475%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:09.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:10.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 53568/249444 objects degraded (21.475%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:10.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:10.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:10.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:10.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:10.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:10.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:10.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:10.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:10.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:10.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:10.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:10.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:10.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:10.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:10.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:10.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:10 smithi017 ceph-mon[107556]: pgmap v969: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 53507/248874 objects degraded (21.500%); 1103/248874 objects misplaced (0.443%); 13 MiB/s, 210 objects/s recovering 2024-09-13T14:42:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:10 smithi017 ceph-mon[103001]: pgmap v969: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 53507/248874 objects degraded (21.500%); 1103/248874 objects misplaced (0.443%); 13 MiB/s, 210 objects/s recovering 2024-09-13T14:42:10.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:10 smithi189 ceph-mon[90992]: pgmap v969: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 53507/248874 objects degraded (21.500%); 1103/248874 objects misplaced (0.443%); 13 MiB/s, 210 objects/s recovering 2024-09-13T14:42:11.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:11.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:11.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:11.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:11.214 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:11.235 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:11.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:11.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:11.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:11.489 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:11.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:11.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:11.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:11.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:11.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:11.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:11.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:11.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:12.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:12.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:12.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:12.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:12.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:12.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:12.360 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:42:12.360 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:12.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:12.386 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:42:12.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:12.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:42:12.409 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:12.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:12.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:42:12.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:12.463 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:42:12.464 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:12.480 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:42:12.480 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:12.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:42:12.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:12.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:12.513 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:42:12.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:12.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:12.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:12.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:12.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:12.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:12.910 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[103001]: pgmap v970: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 53507/248184 objects degraded (21.559%); 150/248184 objects misplaced (0.060%); 18 MiB/s, 287 objects/s recovering 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4032557209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[107556]: pgmap v970: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 53507/248184 objects degraded (21.559%); 150/248184 objects misplaced (0.060%); 18 MiB/s, 287 objects/s recovering 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4032557209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:12 smithi189 ceph-mon[90992]: pgmap v970: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 53507/248184 objects degraded (21.559%); 150/248184 objects misplaced (0.060%); 18 MiB/s, 287 objects/s recovering 2024-09-13T14:42:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4032557209' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:13.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:13.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:13.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:13.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:13.260 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:13.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:13.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:13.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:13.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:13.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:13.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:13.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:13.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:13 smithi017 ceph-mon[103001]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:42:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:13 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 53507/248184 objects degraded (21.559%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:13 smithi017 ceph-mon[107556]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:42:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:13 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 53507/248184 objects degraded (21.559%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:13 smithi189 ceph-mon[90992]: osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:42:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:13 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 53507/248184 objects degraded (21.559%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:14.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:14.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:14.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:14.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:14.278 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:14.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:14.403 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:14.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:14.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:14.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:14.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:14.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:14 smithi017 ceph-mon[103001]: pgmap v972: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 52574/247665 objects degraded (21.228%); 150/247665 objects misplaced (0.061%); 20 MiB/s, 320 objects/s recovering 2024-09-13T14:42:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:14 smithi017 ceph-mon[103001]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:42:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:14 smithi017 ceph-mon[107556]: pgmap v972: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 52574/247665 objects degraded (21.228%); 150/247665 objects misplaced (0.061%); 20 MiB/s, 320 objects/s recovering 2024-09-13T14:42:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:14 smithi017 ceph-mon[107556]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:42:14.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:14.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:14.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:14.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:14 smithi189 ceph-mon[90992]: pgmap v972: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 52574/247665 objects degraded (21.228%); 150/247665 objects misplaced (0.061%); 20 MiB/s, 320 objects/s recovering 2024-09-13T14:42:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:14 smithi189 ceph-mon[90992]: osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:42:15.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:15.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:15.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:15.160 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:15.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:15.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:15.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:15.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:15.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:15.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:15.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:15.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:15.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:15.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:15.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.690 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:15.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:15.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:15.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:15.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:15.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:15.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:16.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:16.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:16.168 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:16.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:16.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:16.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:16.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:16.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:16.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:16.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:16.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:16.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:16.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:16.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:16.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:16 smithi017 ceph-mon[103001]: pgmap v974: 99 pgs: 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 48526/246837 objects degraded (19.659%); 4059/246837 objects misplaced (1.644%); 16 MiB/s, 249 objects/s recovering 2024-09-13T14:42:16.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:16 smithi017 ceph-mon[107556]: pgmap v974: 99 pgs: 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 48526/246837 objects degraded (19.659%); 4059/246837 objects misplaced (1.644%); 16 MiB/s, 249 objects/s recovering 2024-09-13T14:42:16.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:16.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:16 smithi189 ceph-mon[90992]: pgmap v974: 99 pgs: 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 48526/246837 objects degraded (19.659%); 4059/246837 objects misplaced (1.644%); 16 MiB/s, 249 objects/s recovering 2024-09-13T14:42:17.047 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:17.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:17.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:17.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:17.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:17.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:17.304 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:17.304 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:17.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:17.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:17] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:42:17.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:17.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:17.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:17.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:17.655 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:42:17.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:17.704 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:17.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:17.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:17.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/84355356' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:17 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/84355356' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:17 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:17.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/84355356' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:18.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:17 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:42:18.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:18.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:18.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:42:18.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:18.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:18.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:18.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:18.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:18.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:18.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:18.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:18.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:18.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:18.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:18.735 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:18.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:18.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[107556]: pgmap v975: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 47134/246054 objects degraded (19.156%); 3909/246054 objects misplaced (1.589%); 26 MiB/s, 434 objects/s recovering 2024-09-13T14:42:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:42:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[107556]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:42:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 47134/246054 objects degraded (19.156%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:18.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[103001]: pgmap v975: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 47134/246054 objects degraded (19.156%); 3909/246054 objects misplaced (1.589%); 26 MiB/s, 434 objects/s recovering 2024-09-13T14:42:18.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:42:18.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[103001]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:42:18.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:18 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 47134/246054 objects degraded (19.156%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:18.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:18.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:42:18.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:18 smithi189 ceph-mon[90992]: pgmap v975: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 47134/246054 objects degraded (19.156%); 3909/246054 objects misplaced (1.589%); 26 MiB/s, 434 objects/s recovering 2024-09-13T14:42:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:18 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:42:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:18 smithi189 ceph-mon[90992]: osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:42:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:18 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 47134/246054 objects degraded (19.156%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:19.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:19.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:19.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:19.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:19.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:19.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:19.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:19.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:19.585 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:19.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:19.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:19.718 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:19.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:19.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:19.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:19 smithi017 ceph-mon[103001]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:42:19.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:19 smithi017 ceph-mon[107556]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:42:19.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:19.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:19.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:19.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:19 smithi189 ceph-mon[90992]: osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:42:20.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:20.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:20.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:20.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:20.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:20.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:20.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:20.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:20.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:20.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:20.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:20 smithi017 ceph-mon[103001]: pgmap v978: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 41838/245508 objects degraded (17.041%); 8546/245508 objects misplaced (3.481%); 20 MiB/s, 375 objects/s recovering 2024-09-13T14:42:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:20 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:42:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:20 smithi017 ceph-mon[107556]: pgmap v978: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 41838/245508 objects degraded (17.041%); 8546/245508 objects misplaced (3.481%); 20 MiB/s, 375 objects/s recovering 2024-09-13T14:42:20.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:20 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:42:20.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:21.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:20 smithi189 ceph-mon[90992]: pgmap v978: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 41838/245508 objects degraded (17.041%); 8546/245508 objects misplaced (3.481%); 20 MiB/s, 375 objects/s recovering 2024-09-13T14:42:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:20 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:42:21.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:21.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:21.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:21.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:21.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:21.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:21.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:21.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:21.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:21.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:42:21.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:21.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:21.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:21.866 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:21.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:21.990 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:22.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:22.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:22.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:22.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:22.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:22.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:22.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:22.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:22.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:22.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:22.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:22.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:22 smithi017 ceph-mon[103001]: pgmap v979: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41838/244893 objects degraded (17.084%); 8546/244893 objects misplaced (3.490%); 16 MiB/s, 299 objects/s recovering 2024-09-13T14:42:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:22 smithi017 ceph-mon[107556]: pgmap v979: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41838/244893 objects degraded (17.084%); 8546/244893 objects misplaced (3.490%); 16 MiB/s, 299 objects/s recovering 2024-09-13T14:42:23.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:22 smithi189 ceph-mon[90992]: pgmap v979: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41838/244893 objects degraded (17.084%); 8546/244893 objects misplaced (3.490%); 16 MiB/s, 299 objects/s recovering 2024-09-13T14:42:23.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:23.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:23.253 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:23.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:23.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:23.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:23.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:23.522 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:23.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:23.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:23.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:23.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:23.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:23 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41838/244893 objects degraded (17.084%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:23 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41838/244893 objects degraded (17.084%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:24.001 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:24.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:23 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41838/244893 objects degraded (17.084%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:24.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:24.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:24.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:24.249 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:24.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:24.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:24.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:24.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:24.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:24.628 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:24.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:24.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:24.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:24.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:24.808 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:24.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:24.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:24 smithi017 ceph-mon[103001]: pgmap v980: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 41751/244482 objects degraded (17.077%); 8511/244482 objects misplaced (3.481%); 15 MiB/s, 274 objects/s recovering 2024-09-13T14:42:24.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:24 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:42:24.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:24 smithi017 ceph-mon[107556]: pgmap v980: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 41751/244482 objects degraded (17.077%); 8511/244482 objects misplaced (3.481%); 15 MiB/s, 274 objects/s recovering 2024-09-13T14:42:24.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:24 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:42:24.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:24 smithi189 ceph-mon[90992]: pgmap v980: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 41751/244482 objects degraded (17.077%); 8511/244482 objects misplaced (3.481%); 15 MiB/s, 274 objects/s recovering 2024-09-13T14:42:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:24 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:42:25.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:25.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:25.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:25.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:25.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:25.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:25.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:25.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:25.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:25.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:25.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:25.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:25.675 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:42:25.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:25.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:25.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:25.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:26.020 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:26.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:26.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:26.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:26.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:26.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:26.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:26.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:26.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:26.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:26.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:26.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:26.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:26.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:26.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[103001]: pgmap v981: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 41751/244005 objects degraded (17.111%); 8039/244005 objects misplaced (3.295%); 7.8 MiB/s, 144 objects/s recovering 2024-09-13T14:42:26.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:26.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:26.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[103001]: osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:42:26.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[107556]: pgmap v981: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 41751/244005 objects degraded (17.111%); 8039/244005 objects misplaced (3.295%); 7.8 MiB/s, 144 objects/s recovering 2024-09-13T14:42:26.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:26.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:26.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:26 smithi017 ceph-mon[107556]: osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:42:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:26 smithi189 ceph-mon[90992]: pgmap v981: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 41751/244005 objects degraded (17.111%); 8039/244005 objects misplaced (3.295%); 7.8 MiB/s, 144 objects/s recovering 2024-09-13T14:42:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:26 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/375884832' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:26 smithi189 ceph-mon[90992]: osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:42:27.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:27.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:27.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:27.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:27.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:27.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:27.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:27.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:27.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:27.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:27.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:27.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:27.650 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:27] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:42:27.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:27.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:27.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:27.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:27.917 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:27.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:28.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:28.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:28.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:28.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:28.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:28.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:28.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:28.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:28.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:28.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:28.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:28.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:28.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:28 smithi017 ceph-mon[103001]: pgmap v983: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 41713/243645 objects degraded (17.120%); 8039/243645 objects misplaced (3.299%); 6.9 MiB/s, 113 objects/s recovering 2024-09-13T14:42:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:28 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41713/243645 objects degraded (17.120%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:28 smithi017 ceph-mon[107556]: pgmap v983: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 41713/243645 objects degraded (17.120%); 8039/243645 objects misplaced (3.299%); 6.9 MiB/s, 113 objects/s recovering 2024-09-13T14:42:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:28 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41713/243645 objects degraded (17.120%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:28 smithi189 ceph-mon[90992]: pgmap v983: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 41713/243645 objects degraded (17.120%); 8039/243645 objects misplaced (3.299%); 6.9 MiB/s, 113 objects/s recovering 2024-09-13T14:42:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:28 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41713/243645 objects degraded (17.120%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:29.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:29.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:29.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:29.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:29.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:29.400 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:29.400 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:29.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:29.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:29.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:29.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:29.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:29.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:29.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:29.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:29.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:29.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:30.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:30.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:30.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:30.174 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:30.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:30.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.300 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:30.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:30.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:30.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:30.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:30.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:30.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:30.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:30.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:42:30.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:30.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:30.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:30 smithi017 ceph-mon[103001]: pgmap v984: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 41679/243000 objects degraded (17.152%); 7528/243000 objects misplaced (3.098%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:30 smithi017 ceph-mon[107556]: pgmap v984: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 41679/243000 objects degraded (17.152%); 7528/243000 objects misplaced (3.098%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:30.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:30 smithi189 ceph-mon[90992]: pgmap v984: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 83 op/s; 41679/243000 objects degraded (17.152%); 7528/243000 objects misplaced (3.098%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:31.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:42:31.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:31.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:31.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:31.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:31.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.347 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:31.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:31.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:31.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:31.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:31.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:31.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:31.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:31.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:31.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3438227803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:31 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3438227803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:31 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:31.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:31.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3438227803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:31 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:42:32.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:32.197 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:42:32.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:32.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:32.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:32.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:32.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:32.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:32.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:32.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:32.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:32.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:32.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:32.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[103001]: pgmap v985: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 41679/242253 objects degraded (17.205%); 7528/242253 objects misplaced (3.107%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[103001]: osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:42:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[107556]: pgmap v985: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 41679/242253 objects degraded (17.205%); 7528/242253 objects misplaced (3.107%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:32 smithi017 ceph-mon[107556]: osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:42:32.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:32.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:32 smithi189 ceph-mon[90992]: pgmap v985: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 41679/242253 objects degraded (17.205%); 7528/242253 objects misplaced (3.107%); 6.6 MiB/s, 108 objects/s recovering 2024-09-13T14:42:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:32 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:42:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:32 smithi189 ceph-mon[90992]: osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:42:33.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:33.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:33.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:33.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:33.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:33.599 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:33.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:33.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:33.823 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:33.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:33.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:33.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:33 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41679/242253 objects degraded (17.205%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:33 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41679/242253 objects degraded (17.205%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:33.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:34.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:33 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41679/242253 objects degraded (17.205%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:34.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:34.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:34.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:34.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:34.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:34.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:34.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:34.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:34.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:34.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:42:34.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:34.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:34.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:34.729 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:34.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:34.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:34.750 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:34 smithi017 ceph-mon[107556]: pgmap v987: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 41578/241743 objects degraded (17.199%); 7496/241743 objects misplaced (3.101%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:42:34.750 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:34 smithi017 ceph-mon[103001]: pgmap v987: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 41578/241743 objects degraded (17.199%); 7496/241743 objects misplaced (3.101%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:42:34.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:34.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:34.968 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:34.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:34 smithi189 ceph-mon[90992]: pgmap v987: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 41578/241743 objects degraded (17.199%); 7496/241743 objects misplaced (3.101%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:42:35.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:35.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:35.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:35.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:35.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:35.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:35.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:35.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:35.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:35.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:35.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:35.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:35.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:35.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:35.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:35.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:35.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:35.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:35.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:36.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:36.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:36.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:36.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:36.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:36.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:36.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:36.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:36.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:36.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:36 smithi017 ceph-mon[103001]: pgmap v988: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 41578/240867 objects degraded (17.262%); 6975/240867 objects misplaced (2.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:36 smithi017 ceph-mon[107556]: pgmap v988: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 41578/240867 objects degraded (17.262%); 6975/240867 objects misplaced (2.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:37.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:36 smithi189 ceph-mon[90992]: pgmap v988: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 41578/240867 objects degraded (17.262%); 6975/240867 objects misplaced (2.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:37.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:37.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:37.044 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:37.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:37.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:37.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:37.280 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:37.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:37.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:37.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:37] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:42:37.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:37.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:37.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:37.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:37.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:37.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:38.224 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:38.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:38.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:38.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:38.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:38.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:38.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:38.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:38.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:38.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:38.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:38.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:38.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:38.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:38.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:38 smithi017 ceph-mon[103001]: pgmap v989: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 41504/240228 objects degraded (17.277%); 6975/240228 objects misplaced (2.903%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:42:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:38 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41504/240228 objects degraded (17.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:38 smithi017 ceph-mon[107556]: pgmap v989: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 41504/240228 objects degraded (17.277%); 6975/240228 objects misplaced (2.903%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:42:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:38 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41504/240228 objects degraded (17.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:38.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:38 smithi189 ceph-mon[90992]: pgmap v989: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 41504/240228 objects degraded (17.277%); 6975/240228 objects misplaced (2.903%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:42:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:38 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41504/240228 objects degraded (17.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:39.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:39.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:39.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:39.233 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:39.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:39.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:39.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:39.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:39.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:39.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:39.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:39.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:39.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:39.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:39.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:39.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:39.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:39.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:40.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:40.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:40.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:40.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:40.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:40.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:40.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:40.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:40.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:40.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:40.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:40.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:40.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:40.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:40.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:40.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:40.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:40.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:40 smithi017 ceph-mon[103001]: pgmap v990: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41473/239331 objects degraded (17.329%); 6319/239331 objects misplaced (2.640%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:40 smithi017 ceph-mon[107556]: pgmap v990: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41473/239331 objects degraded (17.329%); 6319/239331 objects misplaced (2.640%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:40.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:41.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:40 smithi189 ceph-mon[90992]: pgmap v990: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 41473/239331 objects degraded (17.329%); 6319/239331 objects misplaced (2.640%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:41.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:41.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:41.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:41.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:41.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:41.375 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:41.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:41.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:41.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:41.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:41.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:41.748 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:41.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:41.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:42.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:41 smithi189 ceph-mon[90992]: pgmap v991: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 41473/238332 objects degraded (17.401%); 6319/238332 objects misplaced (2.651%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:42.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:42.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:41 smithi017 ceph-mon[103001]: pgmap v991: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 41473/238332 objects degraded (17.401%); 6319/238332 objects misplaced (2.651%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:41 smithi017 ceph-mon[107556]: pgmap v991: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 41473/238332 objects degraded (17.401%); 6319/238332 objects misplaced (2.651%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:42:42.224 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:42.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:42.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:42.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:42.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:42.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:42.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:42.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:42.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:43.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:43.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:43.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:43.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:43.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:43.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:43.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:43.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:43.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:43.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:43.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:43.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:43.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:43.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:43.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:43.747 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:43.747 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:43.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:43.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:43.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:43 smithi017 ceph-mon[107556]: pgmap v992: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 41366/237771 objects degraded (17.397%); 6286/237771 objects misplaced (2.644%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41473/238332 objects degraded (17.401%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:43 smithi017 ceph-mon[103001]: pgmap v992: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 41366/237771 objects degraded (17.397%); 6286/237771 objects misplaced (2.644%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41473/238332 objects degraded (17.401%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:43.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:43 smithi189 ceph-mon[90992]: pgmap v992: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 41366/237771 objects degraded (17.397%); 6286/237771 objects misplaced (2.644%); 7.3 MiB/s, 117 objects/s recovering 2024-09-13T14:42:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41473/238332 objects degraded (17.401%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:44.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:44.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:44.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:44.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:44.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:44.365 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:44.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:44.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:44.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:44.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:44.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:44.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:44.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:44.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:44.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:45.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:45.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:45.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:45.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:45.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:45.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:45.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.522 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:45.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.624 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.646 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:45.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:45.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:45.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:45.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:46.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:46.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:46.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:42:46.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.148 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:46.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:46.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:46.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:46.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:46.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:46.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:46 smithi017 ceph-mon[103001]: pgmap v993: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 41366/237111 objects degraded (17.446%); 5476/237111 objects misplaced (2.309%); 11 MiB/s, 174 objects/s recovering 2024-09-13T14:42:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:46 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/328261223' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:46 smithi017 ceph-mon[107556]: pgmap v993: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 41366/237111 objects degraded (17.446%); 5476/237111 objects misplaced (2.309%); 11 MiB/s, 174 objects/s recovering 2024-09-13T14:42:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:46 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/328261223' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:46.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:46.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:46 smithi189 ceph-mon[90992]: pgmap v993: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 41366/237111 objects degraded (17.446%); 5476/237111 objects misplaced (2.309%); 11 MiB/s, 174 objects/s recovering 2024-09-13T14:42:46.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:46 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/328261223' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:46.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:46.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:46.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:46.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:46.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:46.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:46.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:47.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:47.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:47.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:47.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:47.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:47.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:47.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:47.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:47.522 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:47] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.51.0" 2024-09-13T14:42:47.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:47.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:47.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:47.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:47.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:47.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:47.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:47.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:47.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:48.016 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:48.040 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:48.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.244 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:42:48.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:48.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:48.391 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:48.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:48.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:42:48.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[103001]: pgmap v994: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 41302/236679 objects degraded (17.451%); 5476/236679 objects misplaced (2.314%); 8.4 MiB/s, 133 objects/s recovering 2024-09-13T14:42:48.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2514868753' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[107556]: pgmap v994: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 41302/236679 objects degraded (17.451%); 5476/236679 objects misplaced (2.314%); 8.4 MiB/s, 133 objects/s recovering 2024-09-13T14:42:48.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2514868753' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:48 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:48 smithi189 ceph-mon[90992]: pgmap v994: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 41302/236679 objects degraded (17.451%); 5476/236679 objects misplaced (2.314%); 8.4 MiB/s, 133 objects/s recovering 2024-09-13T14:42:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2514868753' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:48 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:42:48.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:48.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:48.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:48.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:48.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:48.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:49.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:49.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:42:49.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:49.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:49.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:49.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.297 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:49.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:49.398 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:49.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:49.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:49 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:42:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:49 smithi189 ceph-mon[90992]: osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:42:49.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 41302/236679 objects degraded (17.451%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:49.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:49 smithi189 ceph-mon[90992]: osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:42:49.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:49.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:49.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:49.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:42:49.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[107556]: osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:42:49.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 41302/236679 objects degraded (17.451%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:49.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[107556]: osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:42:49.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:42:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[103001]: osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:42:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 41302/236679 objects degraded (17.451%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[103001]: osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:42:49.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:49.605 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:42:49.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:49.641 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:42:49.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:49.674 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:49.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:49.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:49.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:49.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:50.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:50.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:50.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:50.170 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:50.272 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:50.294 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:50.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:50.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:50.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[103001]: pgmap v997: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 40682/235992 objects degraded (17.239%); 4827/235992 objects misplaced (2.045%); 12 MiB/s, 268 objects/s recovering 2024-09-13T14:42:50.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:50.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:50.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[103001]: osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:42:50.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[107556]: pgmap v997: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 40682/235992 objects degraded (17.239%); 4827/235992 objects misplaced (2.045%); 12 MiB/s, 268 objects/s recovering 2024-09-13T14:42:50.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:50.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:50.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:50 smithi017 ceph-mon[107556]: osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:42:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:42:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:50 smithi189 ceph-mon[90992]: pgmap v997: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 40682/235992 objects degraded (17.239%); 4827/235992 objects misplaced (2.045%); 12 MiB/s, 268 objects/s recovering 2024-09-13T14:42:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:50 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3777858940' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:50 smithi189 ceph-mon[90992]: osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:42:50.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:50.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:50.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:50.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:50.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:50.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:50.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:50.908 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:50.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:51.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:51.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:51.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:51.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:51.196 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:51.197 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:51.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:51.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:51.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:51.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:51.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:51.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:51.647 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:51.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:51.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:51.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:51.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:51.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: pgmap v999: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40682/235206 objects degraded (17.296%); 4827/235206 objects misplaced (2.052%); 7.1 MiB/s, 211 objects/s recovering 2024-09-13T14:42:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: pgmap v999: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40682/235206 objects degraded (17.296%); 4827/235206 objects misplaced (2.052%); 7.1 MiB/s, 211 objects/s recovering 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:51.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:51 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:52.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:52.040 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:52.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:52.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: pgmap v999: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40682/235206 objects degraded (17.296%); 4827/235206 objects misplaced (2.052%); 7.1 MiB/s, 211 objects/s recovering 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:52.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:51 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:42:52.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:52.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:52.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:52.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:52.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:42:52.539 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:52.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:52.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:52.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:52.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:52.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:52.913 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:53.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:53.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:53.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:53.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:53.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:53.209 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:53.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:53.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:53.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:53.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:53.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:53.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:53.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:53.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:54.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:54.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:54.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:54.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:54.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:54.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:54.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:54.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:54 smithi017 ceph-mon[103001]: pgmap v1000: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 40626/234846 objects degraded (17.299%); 4827/234846 objects misplaced (2.055%); 7.1 MiB/s, 210 objects/s recovering 2024-09-13T14:42:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40626/234846 objects degraded (17.299%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:54 smithi017 ceph-mon[107556]: pgmap v1000: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 40626/234846 objects degraded (17.299%); 4827/234846 objects misplaced (2.055%); 7.1 MiB/s, 210 objects/s recovering 2024-09-13T14:42:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40626/234846 objects degraded (17.299%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:54 smithi189 ceph-mon[90992]: pgmap v1000: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 40626/234846 objects degraded (17.299%); 4827/234846 objects misplaced (2.055%); 7.1 MiB/s, 210 objects/s recovering 2024-09-13T14:42:54.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40626/234846 objects degraded (17.299%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:54.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:54.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:54.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:54.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:54.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:54.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:54.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:54.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:55.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:55.038 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:55.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:55.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:55.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:55.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:55.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:55.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:55.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:55.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:55.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:55.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:55.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:55.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:55.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:55.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:55.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:56.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:56.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:56.266 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:56.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:56 smithi017 ceph-mon[103001]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 40591/234018 objects degraded (17.345%); 4791/234018 objects misplaced (2.047%); 6.1 MiB/s, 182 objects/s recovering 2024-09-13T14:42:56.279 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:56 smithi017 ceph-mon[107556]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 40591/234018 objects degraded (17.345%); 4791/234018 objects misplaced (2.047%); 6.1 MiB/s, 182 objects/s recovering 2024-09-13T14:42:56.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:56.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:56.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:56.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:56 smithi189 ceph-mon[90992]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 40591/234018 objects degraded (17.345%); 4791/234018 objects misplaced (2.047%); 6.1 MiB/s, 182 objects/s recovering 2024-09-13T14:42:56.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:56.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:56.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:56.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:56.741 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:42:56.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:56.757 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:42:56.757 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:56.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:56.773 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:42:56.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:56.789 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:56.790 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:42:56.791 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:56.805 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:42:56.805 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:56.821 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:42:56.821 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:56.836 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:42:56.836 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:56.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:42:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:56.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:56.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:57.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:57.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:57.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:57.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:57.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:57.243 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:42:57.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:57.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:57.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:57.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/380221178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:57.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:57.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:57.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:57.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:57.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/380221178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:57.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:57.559 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:42:57] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:42:57.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/380221178' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:57.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:57.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:57.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:57.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:57.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:57.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:57.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:57.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:58.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:58.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:58.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:58.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:58.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[103001]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40519/233562 objects degraded (17.348%); 4791/233562 objects misplaced (2.051%); 5.2 MiB/s, 152 objects/s recovering 2024-09-13T14:42:58.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:58.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:58.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[103001]: osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:42:58.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[107556]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40519/233562 objects degraded (17.348%); 4791/233562 objects misplaced (2.051%); 5.2 MiB/s, 152 objects/s recovering 2024-09-13T14:42:58.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:58.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:58.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:58 smithi017 ceph-mon[107556]: osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:42:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:58 smithi189 ceph-mon[90992]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 40519/233562 objects degraded (17.348%); 4791/233562 objects misplaced (2.051%); 5.2 MiB/s, 152 objects/s recovering 2024-09-13T14:42:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:58 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3497085546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:58 smithi189 ceph-mon[90992]: osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:42:58.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:58.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:58.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:58.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:58.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:58.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:58.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:58.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:58.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:59.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:59.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:59.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:59.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:59.260 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:59.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:59.290 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:59.290 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:59.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:59.390 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:42:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40519/233562 objects degraded (17.348%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:59.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:42:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40519/233562 objects degraded (17.348%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:59.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:42:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40519/233562 objects degraded (17.348%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:42:59.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:59.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:59.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:59.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:59.688 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:42:59.689 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:42:59.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:59.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:42:59.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:59.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:59.910 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:42:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:00.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:43:00.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:00.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:00.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:00.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:00.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:00.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:00.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:00.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:00.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:00.661 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1004,"stamp":"2024-09-13T14:42:59.182184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5090806454,"num_objects":77637,"num_object_clones":0,"num_object_copies":232911,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":40483,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":77637,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":106016,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21543,"num_bytes_recovered":1347848349,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107201,"ondisk_log_size":107201,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18617972,"kb_used_data":13646432,"kb_used_omap":44,"kb_used_meta":4971411,"kb_avail":731343244,"statfs":{"total":767960285184,"available":748895481856,"internally_reserved":0,"allocated":13973946368,"data_stored":13960069620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5090725111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,1,1,2,0,2,2,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-67305472,"num_objects":-1027,"num_object_clones":0,"num_object_copies":-3081,"num_objects_missing_on_primary":-71,"num_objects_missing":-71,"num_objects_degraded":-199,"num_objects_misplaced":-72,"num_objects_unfound":0,"num_objects_dirty":-1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1027,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8912896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.541220"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":559,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.684345+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:42:49.684345+0000","last_peered":"2024-09-13T14:42:49.684345+0000","last_clean":"2024-09-13T14:42:49.684345+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:42:49.684345+0000","last_undegraded":"2024-09-13T14:42:49.684345+0000","last_fullsized":"2024-09-13T14:42:49.684345+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":337,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665679+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:42:49.665679+0000","last_peered":"2024-09-13T14:42:49.665679+0000","last_clean":"2024-09-13T14:42:49.665679+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:42:49.665679+0000","last_undegraded":"2024-09-13T14:42:49.665679+0000","last_fullsized":"2024-09-13T14:42:49.665679+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642910+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:42:49.642910+0000","last_peered":"2024-09-13T14:42:49.642910+0000","last_clean":"2024-09-13T14:42:49.642910+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:42:49.642910+0000","last_undegraded":"2024-09-13T14:42:49.642910+0000","last_fullsized":"2024-09-13T14:42:49.642910+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":546,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246719+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:42:57.246719+0000","last_peered":"2024-09-13T14:42:57.246719+0000","last_clean":"2024-09-13T14:42:57.246719+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:42:57.246719+0000","last_undegraded":"2024-09-13T14:42:57.246719+0000","last_fullsized":"2024-09-13T14:42:57.246719+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":546,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248253+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:42:57.248253+0000","last_peered":"2024-09-13T14:42:57.248253+0000","last_clean":"2024-09-13T14:42:57.248253+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:42:57.248253+0000","last_undegraded":"2024-09-13T14:42:57.248253+0000","last_fullsized":"2024-09-13T14:42:57.248253+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":306,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665311+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:42:49.665311+0000","last_peered":"2024-09-13T14:42:49.665311+0000","last_clean":"2024-09-13T14:42:49.665311+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:42:49.665311+0000","last_undegraded":"2024-09-13T14:42:49.665311+0000","last_fullsized":"2024-09-13T14:42:49.665311+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":558,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.684805+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:42:49.684805+0000","last_peered":"2024-09-13T14:42:49.684805+0000","last_clean":"2024-09-13T14:42:49.684805+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:42:49.684805+0000","last_undegraded":"2024-09-13T14:42:49.684805+0000","last_fullsized":"2024-09-13T14:42:49.684805+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":328,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674447+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:42:49.674447+0000","last_peered":"2024-09-13T14:42:49.674447+0000","last_clean":"2024-09-13T14:42:49.674447+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:42:49.674447+0000","last_undegraded":"2024-09-13T14:42:49.674447+0000","last_fullsized":"2024-09-13T14:42:49.674447+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"338'6486","reported_seq":9085,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:56.251495+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:42:56.251495+0000","last_peered":"2024-09-13T14:42:56.251495+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:42:56.251495+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6486,"log_dups_size":0,"ondisk_log_size":6486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314834944,"num_objects":4804,"num_object_clones":0,"num_object_copies":14412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8523,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6486,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":841,"num_bytes_recovered":55115776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3820}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":332,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666024+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:42:49.666024+0000","last_peered":"2024-09-13T14:42:49.666024+0000","last_clean":"2024-09-13T14:42:49.666024+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:42:49.666024+0000","last_undegraded":"2024-09-13T14:42:49.666024+0000","last_fullsized":"2024-09-13T14:42:49.666024+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642277+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:42:49.642277+0000","last_peered":"2024-09-13T14:42:49.642277+0000","last_clean":"2024-09-13T14:42:49.642277+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:42:49.642277+0000","last_undegraded":"2024-09-13T14:42:49.642277+0000","last_fullsized":"2024-09-13T14:42:49.642277+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"339'6607","reported_seq":16143,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.436103+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:42:57.436103+0000","last_peered":"2024-09-13T14:42:57.436103+0000","last_clean":"2024-09-13T14:42:57.436103+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:42:57.436103+0000","last_undegraded":"2024-09-13T14:42:57.436103+0000","last_fullsized":"2024-09-13T14:42:57.436103+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319881216,"num_objects":4881,"num_object_clones":0,"num_object_copies":14643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6607,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":350,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666313+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:42:49.666313+0000","last_peered":"2024-09-13T14:42:49.666313+0000","last_clean":"2024-09-13T14:42:49.666313+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:42:49.666313+0000","last_undegraded":"2024-09-13T14:42:49.666313+0000","last_fullsized":"2024-09-13T14:42:49.666313+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644092+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:42:49.644092+0000","last_peered":"2024-09-13T14:42:49.644092+0000","last_clean":"2024-09-13T14:42:49.644092+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:42:49.644092+0000","last_undegraded":"2024-09-13T14:42:49.644092+0000","last_fullsized":"2024-09-13T14:42:49.644092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"339'6798","reported_seq":6882,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.938197+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:42:58.938197+0000","last_peered":"2024-09-13T14:42:58.938197+0000","last_clean":"2024-09-13T14:42:58.938197+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:42:58.938197+0000","last_undegraded":"2024-09-13T14:42:58.938197+0000","last_fullsized":"2024-09-13T14:42:58.938197+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6798,"log_dups_size":0,"ondisk_log_size":6798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680000,"num_objects":5000,"num_object_clones":0,"num_object_copies":15000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6798,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667504+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:42:49.667504+0000","last_peered":"2024-09-13T14:42:49.667504+0000","last_clean":"2024-09-13T14:42:49.667504+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:42:49.667504+0000","last_undegraded":"2024-09-13T14:42:49.667504+0000","last_fullsized":"2024-09-13T14:42:49.667504+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":319,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674860+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:42:49.674860+0000","last_peered":"2024-09-13T14:42:49.674860+0000","last_clean":"2024-09-13T14:42:49.674860+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:42:49.674860+0000","last_undegraded":"2024-09-13T14:42:49.674860+0000","last_fullsized":"2024-09-13T14:42:49.674860+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"338'6628","reported_seq":6710,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:53.754895+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:42:53.754895+0000","last_peered":"2024-09-13T14:42:53.754895+0000","last_clean":"2024-09-13T14:42:53.754895+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:42:53.754895+0000","last_undegraded":"2024-09-13T14:42:53.754895+0000","last_fullsized":"2024-09-13T14:42:53.754895+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6628,"log_dups_size":0,"ondisk_log_size":6628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322306048,"num_objects":4918,"num_object_clones":0,"num_object_copies":14754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6628,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642699+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:42:49.642699+0000","last_peered":"2024-09-13T14:42:49.642699+0000","last_clean":"2024-09-13T14:42:49.642699+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:42:49.642699+0000","last_undegraded":"2024-09-13T14:42:49.642699+0000","last_fullsized":"2024-09-13T14:42:49.642699+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":572,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246475+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:42:57.246475+0000","last_peered":"2024-09-13T14:42:57.246475+0000","last_clean":"2024-09-13T14:42:57.246475+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:42:57.246475+0000","last_undegraded":"2024-09-13T14:42:57.246475+0000","last_fullsized":"2024-09-13T14:42:57.246475+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"338'6591","reported_seq":6670,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:55.514008+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:42:55.514008+0000","last_peered":"2024-09-13T14:42:55.514008+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:42:55.514008+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6591,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-762}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":450,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665117+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:42:49.665117+0000","last_peered":"2024-09-13T14:42:49.665117+0000","last_clean":"2024-09-13T14:42:49.665117+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:42:49.665117+0000","last_undegraded":"2024-09-13T14:42:49.665117+0000","last_fullsized":"2024-09-13T14:42:49.665117+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":568,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.684570+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:42:49.684570+0000","last_peered":"2024-09-13T14:42:49.684570+0000","last_clean":"2024-09-13T14:42:49.684570+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:42:49.684570+0000","last_undegraded":"2024-09-13T14:42:49.684570+0000","last_fullsized":"2024-09-13T14:42:49.684570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644232+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:42:49.644232+0000","last_peered":"2024-09-13T14:42:49.644232+0000","last_clean":"2024-09-13T14:42:49.644232+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:42:49.644232+0000","last_undegraded":"2024-09-13T14:42:49.644232+0000","last_fullsized":"2024-09-13T14:42:49.644232+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"338'6324","reported_seq":6406,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.278801+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:42:54.278801+0000","last_peered":"2024-09-13T14:42:54.278801+0000","last_clean":"2024-09-13T14:42:54.278801+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:42:54.278801+0000","last_undegraded":"2024-09-13T14:42:54.278801+0000","last_fullsized":"2024-09-13T14:42:54.278801+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6324,"log_dups_size":0,"ondisk_log_size":6324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312606720,"num_objects":4770,"num_object_clones":0,"num_object_copies":14310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6324,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":154,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642323+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:42:49.642323+0000","last_peered":"2024-09-13T14:42:49.642323+0000","last_clean":"2024-09-13T14:42:49.642323+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:42:49.642323+0000","last_undegraded":"2024-09-13T14:42:49.642323+0000","last_fullsized":"2024-09-13T14:42:49.642323+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":340,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674672+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:42:49.674672+0000","last_peered":"2024-09-13T14:42:49.674672+0000","last_clean":"2024-09-13T14:42:49.674672+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:42:49.674672+0000","last_undegraded":"2024-09-13T14:42:49.674672+0000","last_fullsized":"2024-09-13T14:42:49.674672+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":52,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644599+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:42:49.644599+0000","last_peered":"2024-09-13T14:42:49.644599+0000","last_clean":"2024-09-13T14:42:49.644599+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:42:49.644599+0000","last_undegraded":"2024-09-13T14:42:49.644599+0000","last_fullsized":"2024-09-13T14:42:49.644599+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"339'6554","reported_seq":8800,"reported_epoch":339,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:58.260413+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:42:58.260413+0000","last_peered":"2024-09-13T14:42:58.260413+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:42:58.260413+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":4176,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":848,"num_bytes_recovered":55574528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3801},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.643934+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:42:49.643934+0000","last_peered":"2024-09-13T14:42:49.643934+0000","last_clean":"2024-09-13T14:42:49.643934+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:42:49.643934+0000","last_undegraded":"2024-09-13T14:42:49.643934+0000","last_fullsized":"2024-09-13T14:42:49.643934+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644532+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:42:49.644532+0000","last_peered":"2024-09-13T14:42:49.644532+0000","last_clean":"2024-09-13T14:42:49.644532+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:42:49.644532+0000","last_undegraded":"2024-09-13T14:42:49.644532+0000","last_fullsized":"2024-09-13T14:42:49.644532+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"338'6422","reported_seq":6504,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:54.960646+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:42:54.960646+0000","last_peered":"2024-09-13T14:42:54.960646+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:42:54.960646+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6422,"log_dups_size":0,"ondisk_log_size":6422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310378496,"num_objects":4736,"num_object_clones":0,"num_object_copies":14208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6422,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-730}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":147,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642586+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:42:49.642586+0000","last_peered":"2024-09-13T14:42:49.642586+0000","last_clean":"2024-09-13T14:42:49.642586+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:42:49.642586+0000","last_undegraded":"2024-09-13T14:42:49.642586+0000","last_fullsized":"2024-09-13T14:42:49.642586+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":560,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.692033+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:42:49.692033+0000","last_peered":"2024-09-13T14:42:49.692033+0000","last_clean":"2024-09-13T14:42:49.692033+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:42:49.692033+0000","last_undegraded":"2024-09-13T14:42:49.692033+0000","last_fullsized":"2024-09-13T14:42:49.692033+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":522,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667909+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:42:49.667909+0000","last_peered":"2024-09-13T14:42:49.667909+0000","last_clean":"2024-09-13T14:42:49.667909+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:42:49.667909+0000","last_undegraded":"2024-09-13T14:42:49.667909+0000","last_fullsized":"2024-09-13T14:42:49.667909+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"338'6469","reported_seq":6520,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:57.216646+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:42:57.216646+0000","last_peered":"2024-09-13T14:42:57.216646+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:42:57.216646+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311361536,"num_objects":4751,"num_object_clones":0,"num_object_copies":14253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6469,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-745}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":441,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665726+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:42:49.665726+0000","last_peered":"2024-09-13T14:42:49.665726+0000","last_clean":"2024-09-13T14:42:49.665726+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:42:49.665726+0000","last_undegraded":"2024-09-13T14:42:49.665726+0000","last_fullsized":"2024-09-13T14:42:49.665726+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":863,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642649+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:42:49.642649+0000","last_peered":"2024-09-13T14:42:49.642649+0000","last_clean":"2024-09-13T14:42:49.642649+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:42:49.642649+0000","last_undegraded":"2024-09-13T14:42:49.642649+0000","last_fullsized":"2024-09-13T14:42:49.642649+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":514,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.681856+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:42:49.681856+0000","last_peered":"2024-09-13T14:42:49.681856+0000","last_clean":"2024-09-13T14:42:49.681856+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:42:49.681856+0000","last_undegraded":"2024-09-13T14:42:49.681856+0000","last_fullsized":"2024-09-13T14:42:49.681856+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":305,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674955+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:42:49.674955+0000","last_peered":"2024-09-13T14:42:49.674955+0000","last_clean":"2024-09-13T14:42:49.674955+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:42:49.674955+0000","last_undegraded":"2024-09-13T14:42:49.674955+0000","last_fullsized":"2024-09-13T14:42:49.674955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"338'6643","reported_seq":6723,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:54.948323+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:42:54.948323+0000","last_peered":"2024-09-13T14:42:54.948323+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:42:54.948323+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6643,"log_dups_size":0,"ondisk_log_size":6643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321454080,"num_objects":4905,"num_object_clones":0,"num_object_copies":14715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6643,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-775}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":367,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.668059+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:42:49.668059+0000","last_peered":"2024-09-13T14:42:49.668059+0000","last_clean":"2024-09-13T14:42:49.668059+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:42:49.668059+0000","last_undegraded":"2024-09-13T14:42:49.668059+0000","last_fullsized":"2024-09-13T14:42:49.668059+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":729,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665634+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:42:49.665634+0000","last_peered":"2024-09-13T14:42:49.665634+0000","last_clean":"2024-09-13T14:42:49.665634+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:42:49.665634+0000","last_undegraded":"2024-09-13T14:42:49.665634+0000","last_fullsized":"2024-09-13T14:42:49.665634+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":565,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.247062+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:42:57.247062+0000","last_peered":"2024-09-13T14:42:57.247062+0000","last_clean":"2024-09-13T14:42:57.247062+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:42:57.247062+0000","last_undegraded":"2024-09-13T14:42:57.247062+0000","last_fullsized":"2024-09-13T14:42:57.247062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":569,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248442+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:42:57.248442+0000","last_peered":"2024-09-13T14:42:57.248442+0000","last_clean":"2024-09-13T14:42:57.248442+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:42:57.248442+0000","last_undegraded":"2024-09-13T14:42:57.248442+0000","last_fullsized":"2024-09-13T14:42:57.248442+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"338'6668","reported_seq":16291,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.046696+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:42:55.046696+0000","last_peered":"2024-09-13T14:42:55.046696+0000","last_clean":"2024-09-13T14:42:55.046696+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:42:55.046696+0000","last_undegraded":"2024-09-13T14:42:55.046696+0000","last_fullsized":"2024-09-13T14:42:55.046696+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327417856,"num_objects":4996,"num_object_clones":0,"num_object_copies":14988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6668,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":470,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665116+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:42:49.665116+0000","last_peered":"2024-09-13T14:42:49.665116+0000","last_clean":"2024-09-13T14:42:49.665116+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:42:49.665116+0000","last_undegraded":"2024-09-13T14:42:49.665116+0000","last_fullsized":"2024-09-13T14:42:49.665116+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":531,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.682037+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:42:49.682037+0000","last_peered":"2024-09-13T14:42:49.682037+0000","last_clean":"2024-09-13T14:42:49.682037+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:42:49.682037+0000","last_undegraded":"2024-09-13T14:42:49.682037+0000","last_fullsized":"2024-09-13T14:42:49.682037+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":500,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665463+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:42:49.665463+0000","last_peered":"2024-09-13T14:42:49.665463+0000","last_clean":"2024-09-13T14:42:49.665463+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:42:49.665463+0000","last_undegraded":"2024-09-13T14:42:49.665463+0000","last_fullsized":"2024-09-13T14:42:49.665463+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642930+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:42:49.642930+0000","last_peered":"2024-09-13T14:42:49.642930+0000","last_clean":"2024-09-13T14:42:49.642930+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:42:49.642930+0000","last_undegraded":"2024-09-13T14:42:49.642930+0000","last_fullsized":"2024-09-13T14:42:49.642930+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"338'6407","reported_seq":6488,"reported_epoch":338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:55.560601+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:42:55.560601+0000","last_peered":"2024-09-13T14:42:55.560601+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:42:55.560601+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6407,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-686}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":147,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642379+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:42:49.642379+0000","last_peered":"2024-09-13T14:42:49.642379+0000","last_clean":"2024-09-13T14:42:49.642379+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:42:49.642379+0000","last_undegraded":"2024-09-13T14:42:49.642379+0000","last_fullsized":"2024-09-13T14:42:49.642379+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":954,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642754+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:42:49.642754+0000","last_peered":"2024-09-13T14:42:49.642754+0000","last_clean":"2024-09-13T14:42:49.642754+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:42:49.642754+0000","last_undegraded":"2024-09-13T14:42:49.642754+0000","last_fullsized":"2024-09-13T14:42:49.642754+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644235+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:42:49.644235+0000","last_peered":"2024-09-13T14:42:49.644235+0000","last_clean":"2024-09-13T14:42:49.644235+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:42:49.644235+0000","last_undegraded":"2024-09-13T14:42:49.644235+0000","last_fullsized":"2024-09-13T14:42:49.644235+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":520,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665816+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:42:49.665816+0000","last_peered":"2024-09-13T14:42:49.665816+0000","last_clean":"2024-09-13T14:42:49.665816+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:42:49.665816+0000","last_undegraded":"2024-09-13T14:42:49.665816+0000","last_fullsized":"2024-09-13T14:42:49.665816+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"339'6551","reported_seq":15915,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.984559+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:42:58.984559+0000","last_peered":"2024-09-13T14:42:58.984559+0000","last_clean":"2024-09-13T14:42:58.984559+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:42:58.984559+0000","last_undegraded":"2024-09-13T14:42:58.984559+0000","last_fullsized":"2024-09-13T14:42:58.984559+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315162624,"num_objects":4809,"num_object_clones":0,"num_object_copies":14427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6551,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674640+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:42:49.674640+0000","last_peered":"2024-09-13T14:42:49.674640+0000","last_clean":"2024-09-13T14:42:49.674640+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:42:49.674640+0000","last_undegraded":"2024-09-13T14:42:49.674640+0000","last_fullsized":"2024-09-13T14:42:49.674640+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642796+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:42:49.642796+0000","last_peered":"2024-09-13T14:42:49.642796+0000","last_clean":"2024-09-13T14:42:49.642796+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:42:49.642796+0000","last_undegraded":"2024-09-13T14:42:49.642796+0000","last_fullsized":"2024-09-13T14:42:49.642796+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1099,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666024+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:42:49.666024+0000","last_peered":"2024-09-13T14:42:49.666024+0000","last_clean":"2024-09-13T14:42:49.666024+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:42:49.666024+0000","last_undegraded":"2024-09-13T14:42:49.666024+0000","last_fullsized":"2024-09-13T14:42:49.666024+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":501,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667709+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:42:49.667709+0000","last_peered":"2024-09-13T14:42:49.667709+0000","last_clean":"2024-09-13T14:42:49.667709+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:42:49.667709+0000","last_undegraded":"2024-09-13T14:42:49.667709+0000","last_fullsized":"2024-09-13T14:42:49.667709+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"338'6464","reported_seq":6546,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.547404+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:42:55.547404+0000","last_peered":"2024-09-13T14:42:55.547404+0000","last_clean":"2024-09-13T14:42:55.547404+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:42:55.547404+0000","last_undegraded":"2024-09-13T14:42:55.547404+0000","last_fullsized":"2024-09-13T14:42:55.547404+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310640640,"num_objects":4740,"num_object_clones":0,"num_object_copies":14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6464,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674618+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:42:49.674618+0000","last_peered":"2024-09-13T14:42:49.674618+0000","last_clean":"2024-09-13T14:42:49.674618+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:42:49.674618+0000","last_undegraded":"2024-09-13T14:42:49.674618+0000","last_fullsized":"2024-09-13T14:42:49.674618+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":510,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665926+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:42:49.665926+0000","last_peered":"2024-09-13T14:42:49.665926+0000","last_clean":"2024-09-13T14:42:49.665926+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:42:49.665926+0000","last_undegraded":"2024-09-13T14:42:49.665926+0000","last_fullsized":"2024-09-13T14:42:49.665926+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":733,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667517+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:42:49.667517+0000","last_peered":"2024-09-13T14:42:49.667517+0000","last_clean":"2024-09-13T14:42:49.667517+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:42:49.667517+0000","last_undegraded":"2024-09-13T14:42:49.667517+0000","last_fullsized":"2024-09-13T14:42:49.667517+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":548,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246392+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:42:57.246392+0000","last_peered":"2024-09-13T14:42:57.246392+0000","last_clean":"2024-09-13T14:42:57.246392+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:42:57.246392+0000","last_undegraded":"2024-09-13T14:42:57.246392+0000","last_fullsized":"2024-09-13T14:42:57.246392+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"339'6789","reported_seq":6873,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.521539+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:42:58.521539+0000","last_peered":"2024-09-13T14:42:58.521539+0000","last_clean":"2024-09-13T14:42:58.521539+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:42:58.521539+0000","last_undegraded":"2024-09-13T14:42:58.521539+0000","last_fullsized":"2024-09-13T14:42:58.521539+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6789,"log_dups_size":0,"ondisk_log_size":6789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6789,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644446+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:42:49.644446+0000","last_peered":"2024-09-13T14:42:49.644446+0000","last_clean":"2024-09-13T14:42:49.644446+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:42:49.644446+0000","last_undegraded":"2024-09-13T14:42:49.644446+0000","last_fullsized":"2024-09-13T14:42:49.644446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":546,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.685061+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:42:49.685061+0000","last_peered":"2024-09-13T14:42:49.685061+0000","last_clean":"2024-09-13T14:42:49.685061+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:42:49.685061+0000","last_undegraded":"2024-09-13T14:42:49.685061+0000","last_fullsized":"2024-09-13T14:42:49.685061+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":886,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667159+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:42:49.667159+0000","last_peered":"2024-09-13T14:42:49.667159+0000","last_clean":"2024-09-13T14:42:49.667159+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:42:49.667159+0000","last_undegraded":"2024-09-13T14:42:49.667159+0000","last_fullsized":"2024-09-13T14:42:49.667159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":544,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.691847+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:42:49.691847+0000","last_peered":"2024-09-13T14:42:49.691847+0000","last_clean":"2024-09-13T14:42:49.691847+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:42:49.691847+0000","last_undegraded":"2024-09-13T14:42:49.691847+0000","last_fullsized":"2024-09-13T14:42:49.691847+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"338'6668","reported_seq":18018,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.237778+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:42:57.237778+0000","last_peered":"2024-09-13T14:42:57.237778+0000","last_clean":"2024-09-13T14:42:57.237778+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:42:57.237778+0000","last_undegraded":"2024-09-13T14:42:57.237778+0000","last_fullsized":"2024-09-13T14:42:57.237778+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317521944,"num_objects":4846,"num_object_clones":0,"num_object_copies":14538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4846,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6668,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":398,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.684259+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:42:49.684259+0000","last_peered":"2024-09-13T14:42:49.684259+0000","last_clean":"2024-09-13T14:42:49.684259+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:42:49.684259+0000","last_undegraded":"2024-09-13T14:42:49.684259+0000","last_fullsized":"2024-09-13T14:42:49.684259+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":583,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248136+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:42:57.248136+0000","last_peered":"2024-09-13T14:42:57.248136+0000","last_clean":"2024-09-13T14:42:57.248136+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:42:57.248136+0000","last_undegraded":"2024-09-13T14:42:57.248136+0000","last_fullsized":"2024-09-13T14:42:57.248136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":757,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674935+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:42:49.674935+0000","last_peered":"2024-09-13T14:42:49.674935+0000","last_clean":"2024-09-13T14:42:49.674935+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:42:49.674935+0000","last_undegraded":"2024-09-13T14:42:49.674935+0000","last_fullsized":"2024-09-13T14:42:49.674935+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644014+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:42:49.644014+0000","last_peered":"2024-09-13T14:42:49.644014+0000","last_clean":"2024-09-13T14:42:49.644014+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:42:49.644014+0000","last_undegraded":"2024-09-13T14:42:49.644014+0000","last_fullsized":"2024-09-13T14:42:49.644014+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644250+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:42:49.644250+0000","last_peered":"2024-09-13T14:42:49.644250+0000","last_clean":"2024-09-13T14:42:49.644250+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:42:49.644250+0000","last_undegraded":"2024-09-13T14:42:49.644250+0000","last_fullsized":"2024-09-13T14:42:49.644250+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":48,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644110+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:42:49.644110+0000","last_peered":"2024-09-13T14:42:49.644110+0000","last_clean":"2024-09-13T14:42:49.644110+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:42:49.644110+0000","last_undegraded":"2024-09-13T14:42:49.644110+0000","last_fullsized":"2024-09-13T14:42:49.644110+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":498,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667546+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:42:49.667546+0000","last_peered":"2024-09-13T14:42:49.667546+0000","last_clean":"2024-09-13T14:42:49.667546+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:42:49.667546+0000","last_undegraded":"2024-09-13T14:42:49.667546+0000","last_fullsized":"2024-09-13T14:42:49.667546+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667430+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:42:49.667430+0000","last_peered":"2024-09-13T14:42:49.667430+0000","last_clean":"2024-09-13T14:42:49.667430+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:42:49.667430+0000","last_undegraded":"2024-09-13T14:42:49.667430+0000","last_fullsized":"2024-09-13T14:42:49.667430+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642839+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:42:49.642839+0000","last_peered":"2024-09-13T14:42:49.642839+0000","last_clean":"2024-09-13T14:42:49.642839+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:42:49.642839+0000","last_undegraded":"2024-09-13T14:42:49.642839+0000","last_fullsized":"2024-09-13T14:42:49.642839+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":503,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.681856+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:42:49.681856+0000","last_peered":"2024-09-13T14:42:49.681856+0000","last_clean":"2024-09-13T14:42:49.681856+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:42:49.681856+0000","last_undegraded":"2024-09-13T14:42:49.681856+0000","last_fullsized":"2024-09-13T14:42:49.681856+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":556,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.245899+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:42:57.245899+0000","last_peered":"2024-09-13T14:42:57.245899+0000","last_clean":"2024-09-13T14:42:57.245899+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:42:57.245899+0000","last_undegraded":"2024-09-13T14:42:57.245899+0000","last_fullsized":"2024-09-13T14:42:57.245899+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":559,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.691759+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:42:49.691759+0000","last_peered":"2024-09-13T14:42:49.691759+0000","last_clean":"2024-09-13T14:42:49.691759+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:42:49.691759+0000","last_undegraded":"2024-09-13T14:42:49.691759+0000","last_fullsized":"2024-09-13T14:42:49.691759+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":336,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674714+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:42:49.674714+0000","last_peered":"2024-09-13T14:42:49.674714+0000","last_clean":"2024-09-13T14:42:49.674714+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:42:49.674714+0000","last_undegraded":"2024-09-13T14:42:49.674714+0000","last_fullsized":"2024-09-13T14:42:49.674714+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":576,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.691939+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:42:49.691939+0000","last_peered":"2024-09-13T14:42:49.691939+0000","last_clean":"2024-09-13T14:42:49.691939+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:42:49.691939+0000","last_undegraded":"2024-09-13T14:42:49.691939+0000","last_fullsized":"2024-09-13T14:42:49.691939+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":548,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248343+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:42:57.248343+0000","last_peered":"2024-09-13T14:42:57.248343+0000","last_clean":"2024-09-13T14:42:57.248343+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:42:57.248343+0000","last_undegraded":"2024-09-13T14:42:57.248343+0000","last_fullsized":"2024-09-13T14:42:57.248343+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":523,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665510+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:42:49.665510+0000","last_peered":"2024-09-13T14:42:49.665510+0000","last_clean":"2024-09-13T14:42:49.665510+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:42:49.665510+0000","last_undegraded":"2024-09-13T14:42:49.665510+0000","last_fullsized":"2024-09-13T14:42:49.665510+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":572,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.683751+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:42:49.683751+0000","last_peered":"2024-09-13T14:42:49.683751+0000","last_clean":"2024-09-13T14:42:49.683751+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:42:49.683751+0000","last_undegraded":"2024-09-13T14:42:49.683751+0000","last_fullsized":"2024-09-13T14:42:49.683751+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":37,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.655001+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:42:49.655001+0000","last_peered":"2024-09-13T14:42:49.655001+0000","last_clean":"2024-09-13T14:42:49.655001+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:42:49.655001+0000","last_undegraded":"2024-09-13T14:42:49.655001+0000","last_fullsized":"2024-09-13T14:42:49.655001+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":469,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666164+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:42:49.666164+0000","last_peered":"2024-09-13T14:42:49.666164+0000","last_clean":"2024-09-13T14:42:49.666164+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:42:49.666164+0000","last_undegraded":"2024-09-13T14:42:49.666164+0000","last_fullsized":"2024-09-13T14:42:49.666164+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":552,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246290+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:42:57.246290+0000","last_peered":"2024-09-13T14:42:57.246290+0000","last_clean":"2024-09-13T14:42:57.246290+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:42:57.246290+0000","last_undegraded":"2024-09-13T14:42:57.246290+0000","last_fullsized":"2024-09-13T14:42:57.246290+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644184+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:42:49.644184+0000","last_peered":"2024-09-13T14:42:49.644184+0000","last_clean":"2024-09-13T14:42:49.644184+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:42:49.644184+0000","last_undegraded":"2024-09-13T14:42:49.644184+0000","last_fullsized":"2024-09-13T14:42:49.644184+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642759+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:42:49.642759+0000","last_peered":"2024-09-13T14:42:49.642759+0000","last_clean":"2024-09-13T14:42:49.642759+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:42:49.642759+0000","last_undegraded":"2024-09-13T14:42:49.642759+0000","last_fullsized":"2024-09-13T14:42:49.642759+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.654955+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:42:49.654955+0000","last_peered":"2024-09-13T14:42:49.654955+0000","last_clean":"2024-09-13T14:42:49.654955+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:42:49.654955+0000","last_undegraded":"2024-09-13T14:42:49.654955+0000","last_fullsized":"2024-09-13T14:42:49.654955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":270,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642988+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:42:49.642988+0000","last_peered":"2024-09-13T14:42:49.642988+0000","last_clean":"2024-09-13T14:42:49.642988+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:42:49.642988+0000","last_undegraded":"2024-09-13T14:42:49.642988+0000","last_fullsized":"2024-09-13T14:42:49.642988+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":568,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.682124+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:42:49.682124+0000","last_peered":"2024-09-13T14:42:49.682124+0000","last_clean":"2024-09-13T14:42:49.682124+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:42:49.682124+0000","last_undegraded":"2024-09-13T14:42:49.682124+0000","last_fullsized":"2024-09-13T14:42:49.682124+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":555,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.684140+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:42:49.684140+0000","last_peered":"2024-09-13T14:42:49.684140+0000","last_clean":"2024-09-13T14:42:49.684140+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:42:49.684140+0000","last_undegraded":"2024-09-13T14:42:49.684140+0000","last_fullsized":"2024-09-13T14:42:49.684140+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.643904+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:42:49.643904+0000","last_peered":"2024-09-13T14:42:49.643904+0000","last_clean":"2024-09-13T14:42:49.643904+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:42:49.643904+0000","last_undegraded":"2024-09-13T14:42:49.643904+0000","last_fullsized":"2024-09-13T14:42:49.643904+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":562,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.245841+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:42:57.245841+0000","last_peered":"2024-09-13T14:42:57.245841+0000","last_clean":"2024-09-13T14:42:57.245841+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:42:57.245841+0000","last_undegraded":"2024-09-13T14:42:57.245841+0000","last_fullsized":"2024-09-13T14:42:57.245841+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5086117912,"num_objects":77609,"num_object_clones":0,"num_object_copies":232827,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":40483,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":77609,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105069,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21486,"num_bytes_recovered":1347682304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13916123136,"data_stored":13916110920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105069,"ondisk_log_size":105069,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397882,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2742512,"kb_used_data":1854820,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":91002640,"statfs":{"total":95995035648,"available":93186703360,"internally_reserved":0,"allocated":1899335680,"data_stored":1897582588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594110,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2497116,"kb_used_data":1911376,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91248036,"statfs":{"total":95995035648,"available":93437988864,"internally_reserved":0,"allocated":1957249024,"data_stored":1955515686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665564,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":698268,"kb_used_data":361552,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93046884,"statfs":{"total":95995035648,"available":95280009216,"internally_reserved":0,"allocated":370229248,"data_stored":368492241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986566,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969832,"kb_used_data":1074140,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":91775320,"statfs":{"total":95995035648,"available":93977927680,"internally_reserved":0,"allocated":1099919360,"data_stored":1098174366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,2,0,2,2,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697125,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2956868,"kb_used_data":2238972,"kb_used_omap":4,"kb_used_meta":717883,"kb_avail":90788284,"statfs":{"total":95995035648,"available":92967202816,"internally_reserved":0,"allocated":2292707328,"data_stored":2290982321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":735112267},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860652,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2776716,"kb_used_data":2155460,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":90968436,"statfs":{"total":95995035648,"available":93151678464,"internally_reserved":0,"allocated":2207191040,"data_stored":2205458379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020323,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2485760,"kb_used_data":2182200,"kb_used_omap":9,"kb_used_meta":303542,"kb_avail":91259392,"statfs":{"total":95995035648,"available":93449617408,"internally_reserved":0,"allocated":2234572800,"data_stored":2232851266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":310827845},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187714,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490900,"kb_used_data":1867912,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":91254252,"statfs":{"total":95995035648,"available":93444354048,"internally_reserved":0,"allocated":1912741888,"data_stored":1911012773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1903362048,"data_stored":1903362048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2224881664,"data_stored":2224881664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2202603520,"data_stored":2202599448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2284453888,"data_stored":2284453888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1094713344,"data_stored":1094713344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":365301760,"data_stored":365297688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1952776192,"data_stored":1952776192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1888030720,"data_stored":1888026648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:00.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:43:00.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:00.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:00.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:00.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:00 smithi017 ceph-mon[103001]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 40483/232911 objects degraded (17.381%); 4755/232911 objects misplaced (2.042%); 912 KiB/s, 14 objects/s recovering 2024-09-13T14:43:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:00 smithi017 ceph-mon[103001]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:00 smithi017 ceph-mon[107556]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 40483/232911 objects degraded (17.381%); 4755/232911 objects misplaced (2.042%); 912 KiB/s, 14 objects/s recovering 2024-09-13T14:43:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:00 smithi017 ceph-mon[107556]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:00.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:00.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:01.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:00 smithi189 ceph-mon[90992]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 40483/232911 objects degraded (17.381%); 4755/232911 objects misplaced (2.042%); 912 KiB/s, 14 objects/s recovering 2024-09-13T14:43:01.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:00 smithi189 ceph-mon[90992]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:01.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:01.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:43:01.137 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:01.172 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:01.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:01.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:01.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:01.351 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:01.384 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:01.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:01.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:01.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.957 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:01.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:01.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:02.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:02.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:02.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:02.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:02.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:02.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:02.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:02.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:02.596 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:02.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:02.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:02.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:02.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:02.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:02.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:02 smithi017 ceph-mon[103001]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 40483/231933 objects degraded (17.455%); 4755/231933 objects misplaced (2.050%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:02 smithi017 ceph-mon[107556]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 40483/231933 objects degraded (17.455%); 4755/231933 objects misplaced (2.050%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:02.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:02.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:02 smithi189 ceph-mon[90992]: pgmap v1005: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 40483/231933 objects degraded (17.455%); 4755/231933 objects misplaced (2.050%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:03.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:03.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:03.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:03.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:03.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:03.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:03.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:03.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:03.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:03.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:03.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:03.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:03.665 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:03.666 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:03 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40483/231933 objects degraded (17.455%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:03 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40483/231933 objects degraded (17.455%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:03.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:03 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40483/231933 objects degraded (17.455%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:03.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:03.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:03.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:04.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:04.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:04.211 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:04.356 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:04.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:43:04.458 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:04.489 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:43:04.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.602 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.710 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:04.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:04.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:04.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:04 smithi017 ceph-mon[103001]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 40407/231417 objects degraded (17.461%); 4755/231417 objects misplaced (2.055%); 934 KiB/s, 14 objects/s recovering 2024-09-13T14:43:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:04 smithi017 ceph-mon[107556]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 40407/231417 objects degraded (17.461%); 4755/231417 objects misplaced (2.055%); 934 KiB/s, 14 objects/s recovering 2024-09-13T14:43:04.975 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:04 smithi189 ceph-mon[90992]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 40407/231417 objects degraded (17.461%); 4755/231417 objects misplaced (2.055%); 934 KiB/s, 14 objects/s recovering 2024-09-13T14:43:04.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:05.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:05.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:05.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:05.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:05.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:05.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:05.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:05.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:05.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:05.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:05.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:05.897 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:05 smithi189 ceph-mon[90992]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:05 smithi017 ceph-mon[103001]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:05 smithi017 ceph-mon[107556]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:05.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:05.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:06.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:06.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:06.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:06.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:06.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:06.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:06.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:06.453 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:06.454 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:06.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:06.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:06.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:06.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:06.661 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:43:06.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:06.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:06 smithi189 ceph-mon[90992]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40372/230415 objects degraded (17.521%); 4720/230415 objects misplaced (2.048%); 928 KiB/s, 14 objects/s recovering 2024-09-13T14:43:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:06.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:06.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:06 smithi017 ceph-mon[103001]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40372/230415 objects degraded (17.521%); 4720/230415 objects misplaced (2.048%); 928 KiB/s, 14 objects/s recovering 2024-09-13T14:43:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:06 smithi017 ceph-mon[107556]: pgmap v1007: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40372/230415 objects degraded (17.521%); 4720/230415 objects misplaced (2.048%); 928 KiB/s, 14 objects/s recovering 2024-09-13T14:43:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:06.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:07.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:07.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:07.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:07.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:07.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:43:07.148 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:07.148 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:07.157 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:07.157 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:07.173 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:07.174 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:07.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:07.191 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:07.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:07.206 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:07.221 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:07.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:07.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:07.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:07.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:07.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:07.255 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:07.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:07.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:07.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:07.400 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:07.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:07.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:07.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:07.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:07.662 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:07] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:43:07.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:07.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:07.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:07.820 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:07 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:07.821 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:07.821 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:07 smithi189 ceph-mon[90992]: osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:43:07.821 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1792422797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:07.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:07.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:07.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[103001]: osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1792422797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:07.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3536147121' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:07.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[107556]: osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:43:07.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1792422797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:07.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:08.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:08.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:08.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:08.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:08.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:08.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:08.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:08.499 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1006,"stamp":"2024-09-13T14:43:03.183412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5058169526,"num_objects":77139,"num_object_clones":0,"num_object_copies":231417,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":40407,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":77139,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":106514,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21581,"num_bytes_recovered":1350338717,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107699,"ondisk_log_size":107699,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18536856,"kb_used_data":13561168,"kb_used_omap":44,"kb_used_meta":4975571,"kb_avail":731424360,"statfs":{"total":767960285184,"available":748978544640,"internally_reserved":0,"allocated":13886636032,"data_stored":13872735743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5094984951},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1,2,0,2,2,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-74907648,"num_objects":-1143,"num_object_clones":0,"num_object_copies":-3429,"num_objects_missing_on_primary":-71,"num_objects_missing":-71,"num_objects_degraded":-219,"num_objects_misplaced":-72,"num_objects_unfound":0,"num_objects_dirty":-1143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1143,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002871"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":561,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.267259+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:42:57.267259+0000","last_peered":"2024-09-13T14:42:57.267259+0000","last_clean":"2024-09-13T14:42:57.267259+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:42:57.267259+0000","last_undegraded":"2024-09-13T14:42:57.267259+0000","last_fullsized":"2024-09-13T14:42:57.267259+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":337,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665679+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:42:49.665679+0000","last_peered":"2024-09-13T14:42:49.665679+0000","last_clean":"2024-09-13T14:42:49.665679+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:42:49.665679+0000","last_undegraded":"2024-09-13T14:42:49.665679+0000","last_fullsized":"2024-09-13T14:42:49.665679+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642910+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:42:49.642910+0000","last_peered":"2024-09-13T14:42:49.642910+0000","last_clean":"2024-09-13T14:42:49.642910+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:42:49.642910+0000","last_undegraded":"2024-09-13T14:42:49.642910+0000","last_fullsized":"2024-09-13T14:42:49.642910+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":546,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246719+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:42:57.246719+0000","last_peered":"2024-09-13T14:42:57.246719+0000","last_clean":"2024-09-13T14:42:57.246719+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:42:57.246719+0000","last_undegraded":"2024-09-13T14:42:57.246719+0000","last_fullsized":"2024-09-13T14:42:57.246719+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":546,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248253+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:42:57.248253+0000","last_peered":"2024-09-13T14:42:57.248253+0000","last_clean":"2024-09-13T14:42:57.248253+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:42:57.248253+0000","last_undegraded":"2024-09-13T14:42:57.248253+0000","last_fullsized":"2024-09-13T14:42:57.248253+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":306,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665311+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:42:49.665311+0000","last_peered":"2024-09-13T14:42:49.665311+0000","last_clean":"2024-09-13T14:42:49.665311+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:42:49.665311+0000","last_undegraded":"2024-09-13T14:42:49.665311+0000","last_fullsized":"2024-09-13T14:42:49.665311+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":560,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.266108+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:42:57.266108+0000","last_peered":"2024-09-13T14:42:57.266108+0000","last_clean":"2024-09-13T14:42:57.266108+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:42:57.266108+0000","last_undegraded":"2024-09-13T14:42:57.266108+0000","last_fullsized":"2024-09-13T14:42:57.266108+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":328,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674447+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:42:49.674447+0000","last_peered":"2024-09-13T14:42:49.674447+0000","last_clean":"2024-09-13T14:42:49.674447+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:42:49.674447+0000","last_undegraded":"2024-09-13T14:42:49.674447+0000","last_fullsized":"2024-09-13T14:42:49.674447+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"339'6524","reported_seq":9239,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:01.913249+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:43:01.913249+0000","last_peered":"2024-09-13T14:43:01.913249+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:43:01.913249+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312344576,"num_objects":4766,"num_object_clones":0,"num_object_copies":14298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8447,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6524,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":879,"num_bytes_recovered":57606144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3782}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":332,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666024+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:42:49.666024+0000","last_peered":"2024-09-13T14:42:49.666024+0000","last_clean":"2024-09-13T14:42:49.666024+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:42:49.666024+0000","last_undegraded":"2024-09-13T14:42:49.666024+0000","last_fullsized":"2024-09-13T14:42:49.666024+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642277+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:42:49.642277+0000","last_peered":"2024-09-13T14:42:49.642277+0000","last_clean":"2024-09-13T14:42:49.642277+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:42:49.642277+0000","last_undegraded":"2024-09-13T14:42:49.642277+0000","last_fullsized":"2024-09-13T14:42:49.642277+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"339'6649","reported_seq":16185,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:43:02.922355+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:43:02.922355+0000","last_peered":"2024-09-13T14:43:02.922355+0000","last_clean":"2024-09-13T14:43:02.922355+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:43:02.922355+0000","last_undegraded":"2024-09-13T14:43:02.922355+0000","last_fullsized":"2024-09-13T14:43:02.922355+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317128704,"num_objects":4839,"num_object_clones":0,"num_object_copies":14517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":350,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.666313+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:42:49.666313+0000","last_peered":"2024-09-13T14:42:49.666313+0000","last_clean":"2024-09-13T14:42:49.666313+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:42:49.666313+0000","last_undegraded":"2024-09-13T14:42:49.666313+0000","last_fullsized":"2024-09-13T14:42:49.666313+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644092+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:42:49.644092+0000","last_peered":"2024-09-13T14:42:49.644092+0000","last_clean":"2024-09-13T14:42:49.644092+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:42:49.644092+0000","last_undegraded":"2024-09-13T14:42:49.644092+0000","last_fullsized":"2024-09-13T14:42:49.644092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"339'6798","reported_seq":6882,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.938197+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:42:58.938197+0000","last_peered":"2024-09-13T14:42:58.938197+0000","last_clean":"2024-09-13T14:42:58.938197+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:42:58.938197+0000","last_undegraded":"2024-09-13T14:42:58.938197+0000","last_fullsized":"2024-09-13T14:42:58.938197+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6798,"log_dups_size":0,"ondisk_log_size":6798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680000,"num_objects":5000,"num_object_clones":0,"num_object_copies":15000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6798,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667504+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:42:49.667504+0000","last_peered":"2024-09-13T14:42:49.667504+0000","last_clean":"2024-09-13T14:42:49.667504+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:42:49.667504+0000","last_undegraded":"2024-09-13T14:42:49.667504+0000","last_fullsized":"2024-09-13T14:42:49.667504+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":319,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674860+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:42:49.674860+0000","last_peered":"2024-09-13T14:42:49.674860+0000","last_clean":"2024-09-13T14:42:49.674860+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:42:49.674860+0000","last_undegraded":"2024-09-13T14:42:49.674860+0000","last_fullsized":"2024-09-13T14:42:49.674860+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"339'6683","reported_seq":6767,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.713795+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:42:59.713795+0000","last_peered":"2024-09-13T14:42:59.713795+0000","last_clean":"2024-09-13T14:42:59.713795+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:42:59.713795+0000","last_undegraded":"2024-09-13T14:42:59.713795+0000","last_fullsized":"2024-09-13T14:42:59.713795+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318701568,"num_objects":4863,"num_object_clones":0,"num_object_copies":14589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6683,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642699+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:42:49.642699+0000","last_peered":"2024-09-13T14:42:49.642699+0000","last_clean":"2024-09-13T14:42:49.642699+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:42:49.642699+0000","last_undegraded":"2024-09-13T14:42:49.642699+0000","last_fullsized":"2024-09-13T14:42:49.642699+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":572,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246475+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:42:57.246475+0000","last_peered":"2024-09-13T14:42:57.246475+0000","last_clean":"2024-09-13T14:42:57.246475+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:42:57.246475+0000","last_undegraded":"2024-09-13T14:42:57.246475+0000","last_fullsized":"2024-09-13T14:42:57.246475+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"339'6625","reported_seq":6706,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:00.470076+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:43:00.470076+0000","last_peered":"2024-09-13T14:43:00.470076+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:43:00.470076+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6625,"log_dups_size":0,"ondisk_log_size":6625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6625,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-796}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":450,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665117+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:42:49.665117+0000","last_peered":"2024-09-13T14:42:49.665117+0000","last_clean":"2024-09-13T14:42:49.665117+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:42:49.665117+0000","last_undegraded":"2024-09-13T14:42:49.665117+0000","last_fullsized":"2024-09-13T14:42:49.665117+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":570,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265814+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:42:57.265814+0000","last_peered":"2024-09-13T14:42:57.265814+0000","last_clean":"2024-09-13T14:42:57.265814+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:42:57.265814+0000","last_undegraded":"2024-09-13T14:42:57.265814+0000","last_fullsized":"2024-09-13T14:42:57.265814+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644232+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:42:49.644232+0000","last_peered":"2024-09-13T14:42:49.644232+0000","last_clean":"2024-09-13T14:42:49.644232+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:42:49.644232+0000","last_undegraded":"2024-09-13T14:42:49.644232+0000","last_fullsized":"2024-09-13T14:42:49.644232+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"339'6367","reported_seq":6451,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:59.979840+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:42:59.979840+0000","last_peered":"2024-09-13T14:42:59.979840+0000","last_clean":"2024-09-13T14:42:59.979840+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:42:59.979840+0000","last_undegraded":"2024-09-13T14:42:59.979840+0000","last_fullsized":"2024-09-13T14:42:59.979840+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6367,"log_dups_size":0,"ondisk_log_size":6367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309788672,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6367,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":154,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642323+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:42:49.642323+0000","last_peered":"2024-09-13T14:42:49.642323+0000","last_clean":"2024-09-13T14:42:49.642323+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:42:49.642323+0000","last_undegraded":"2024-09-13T14:42:49.642323+0000","last_fullsized":"2024-09-13T14:42:49.642323+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":340,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674672+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:42:49.674672+0000","last_peered":"2024-09-13T14:42:49.674672+0000","last_clean":"2024-09-13T14:42:49.674672+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:42:49.674672+0000","last_undegraded":"2024-09-13T14:42:49.674672+0000","last_fullsized":"2024-09-13T14:42:49.674672+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":52,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644599+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:42:49.644599+0000","last_peered":"2024-09-13T14:42:49.644599+0000","last_clean":"2024-09-13T14:42:49.644599+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:42:49.644599+0000","last_undegraded":"2024-09-13T14:42:49.644599+0000","last_fullsized":"2024-09-13T14:42:49.644599+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"339'6554","reported_seq":8800,"reported_epoch":339,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:58.260413+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:42:58.260413+0000","last_peered":"2024-09-13T14:42:58.260413+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:42:58.260413+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":4176,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":848,"num_bytes_recovered":55574528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3801},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.643934+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:42:49.643934+0000","last_peered":"2024-09-13T14:42:49.643934+0000","last_clean":"2024-09-13T14:42:49.643934+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:42:49.643934+0000","last_undegraded":"2024-09-13T14:42:49.643934+0000","last_fullsized":"2024-09-13T14:42:49.643934+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644532+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:42:49.644532+0000","last_peered":"2024-09-13T14:42:49.644532+0000","last_clean":"2024-09-13T14:42:49.644532+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:42:49.644532+0000","last_undegraded":"2024-09-13T14:42:49.644532+0000","last_fullsized":"2024-09-13T14:42:49.644532+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"339'6453","reported_seq":6537,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:59.897512+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:42:59.897512+0000","last_peered":"2024-09-13T14:42:59.897512+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:42:59.897512+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6453,"log_dups_size":0,"ondisk_log_size":6453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6453,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-761}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":147,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642586+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:42:49.642586+0000","last_peered":"2024-09-13T14:42:49.642586+0000","last_clean":"2024-09-13T14:42:49.642586+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:42:49.642586+0000","last_undegraded":"2024-09-13T14:42:49.642586+0000","last_fullsized":"2024-09-13T14:42:49.642586+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":562,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.272559+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:42:57.272559+0000","last_peered":"2024-09-13T14:42:57.272559+0000","last_clean":"2024-09-13T14:42:57.272559+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:42:57.272559+0000","last_undegraded":"2024-09-13T14:42:57.272559+0000","last_fullsized":"2024-09-13T14:42:57.272559+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":524,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429806+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:42:57.429806+0000","last_peered":"2024-09-13T14:42:57.429806+0000","last_clean":"2024-09-13T14:42:57.429806+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:42:57.429806+0000","last_undegraded":"2024-09-13T14:42:57.429806+0000","last_fullsized":"2024-09-13T14:42:57.429806+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"339'6510","reported_seq":6561,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:02.880031+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:43:02.880031+0000","last_peered":"2024-09-13T14:43:02.880031+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:43:02.880031+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6510,"log_dups_size":0,"ondisk_log_size":6510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308674560,"num_objects":4710,"num_object_clones":0,"num_object_copies":14130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6510,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-786}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":441,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665726+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:42:49.665726+0000","last_peered":"2024-09-13T14:42:49.665726+0000","last_clean":"2024-09-13T14:42:49.665726+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:42:49.665726+0000","last_undegraded":"2024-09-13T14:42:49.665726+0000","last_fullsized":"2024-09-13T14:42:49.665726+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":863,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642649+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:42:49.642649+0000","last_peered":"2024-09-13T14:42:49.642649+0000","last_clean":"2024-09-13T14:42:49.642649+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:42:49.642649+0000","last_undegraded":"2024-09-13T14:42:49.642649+0000","last_fullsized":"2024-09-13T14:42:49.642649+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":516,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265284+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:42:57.265284+0000","last_peered":"2024-09-13T14:42:57.265284+0000","last_clean":"2024-09-13T14:42:57.265284+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:42:57.265284+0000","last_undegraded":"2024-09-13T14:42:57.265284+0000","last_fullsized":"2024-09-13T14:42:57.265284+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":305,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674955+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:42:49.674955+0000","last_peered":"2024-09-13T14:42:49.674955+0000","last_clean":"2024-09-13T14:42:49.674955+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:42:49.674955+0000","last_undegraded":"2024-09-13T14:42:49.674955+0000","last_fullsized":"2024-09-13T14:42:49.674955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"339'6677","reported_seq":6759,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:42:59.975890+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:42:59.975890+0000","last_peered":"2024-09-13T14:42:59.975890+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:42:59.975890+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6677,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-809}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":369,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429518+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:42:57.429518+0000","last_peered":"2024-09-13T14:42:57.429518+0000","last_clean":"2024-09-13T14:42:57.429518+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:42:57.429518+0000","last_undegraded":"2024-09-13T14:42:57.429518+0000","last_fullsized":"2024-09-13T14:42:57.429518+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":729,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665634+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:42:49.665634+0000","last_peered":"2024-09-13T14:42:49.665634+0000","last_clean":"2024-09-13T14:42:49.665634+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:42:49.665634+0000","last_undegraded":"2024-09-13T14:42:49.665634+0000","last_fullsized":"2024-09-13T14:42:49.665634+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":565,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.247062+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:42:57.247062+0000","last_peered":"2024-09-13T14:42:57.247062+0000","last_clean":"2024-09-13T14:42:57.247062+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:42:57.247062+0000","last_undegraded":"2024-09-13T14:42:57.247062+0000","last_fullsized":"2024-09-13T14:42:57.247062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":569,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248442+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:42:57.248442+0000","last_peered":"2024-09-13T14:42:57.248442+0000","last_clean":"2024-09-13T14:42:57.248442+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:42:57.248442+0000","last_undegraded":"2024-09-13T14:42:57.248442+0000","last_fullsized":"2024-09-13T14:42:57.248442+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"339'6716","reported_seq":16341,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:43:00.454531+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:43:00.454531+0000","last_peered":"2024-09-13T14:43:00.454531+0000","last_clean":"2024-09-13T14:43:00.454531+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:43:00.454531+0000","last_undegraded":"2024-09-13T14:43:00.454531+0000","last_fullsized":"2024-09-13T14:43:00.454531+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324272128,"num_objects":4948,"num_object_clones":0,"num_object_copies":14844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6716,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":470,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.665116+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:42:49.665116+0000","last_peered":"2024-09-13T14:42:49.665116+0000","last_clean":"2024-09-13T14:42:49.665116+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:42:49.665116+0000","last_undegraded":"2024-09-13T14:42:49.665116+0000","last_fullsized":"2024-09-13T14:42:49.665116+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":533,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265584+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:42:57.265584+0000","last_peered":"2024-09-13T14:42:57.265584+0000","last_clean":"2024-09-13T14:42:57.265584+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:42:57.265584+0000","last_undegraded":"2024-09-13T14:42:57.265584+0000","last_fullsized":"2024-09-13T14:42:57.265584+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":502,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429235+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:42:57.429235+0000","last_peered":"2024-09-13T14:42:57.429235+0000","last_clean":"2024-09-13T14:42:57.429235+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:42:57.429235+0000","last_undegraded":"2024-09-13T14:42:57.429235+0000","last_fullsized":"2024-09-13T14:42:57.429235+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642930+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:42:49.642930+0000","last_peered":"2024-09-13T14:42:49.642930+0000","last_clean":"2024-09-13T14:42:49.642930+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:42:49.642930+0000","last_undegraded":"2024-09-13T14:42:49.642930+0000","last_fullsized":"2024-09-13T14:42:49.642930+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"339'6447","reported_seq":6530,"reported_epoch":339,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:00.577547+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:43:00.577547+0000","last_peered":"2024-09-13T14:43:00.577547+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:43:00.577547+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313458688,"num_objects":4783,"num_object_clones":0,"num_object_copies":14349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6447,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-726}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":147,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642379+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:42:49.642379+0000","last_peered":"2024-09-13T14:42:49.642379+0000","last_clean":"2024-09-13T14:42:49.642379+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:42:49.642379+0000","last_undegraded":"2024-09-13T14:42:49.642379+0000","last_fullsized":"2024-09-13T14:42:49.642379+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":954,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642754+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:42:49.642754+0000","last_peered":"2024-09-13T14:42:49.642754+0000","last_clean":"2024-09-13T14:42:49.642754+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:42:49.642754+0000","last_undegraded":"2024-09-13T14:42:49.642754+0000","last_fullsized":"2024-09-13T14:42:49.642754+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644235+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:42:49.644235+0000","last_peered":"2024-09-13T14:42:49.644235+0000","last_clean":"2024-09-13T14:42:49.644235+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:42:49.644235+0000","last_undegraded":"2024-09-13T14:42:49.644235+0000","last_fullsized":"2024-09-13T14:42:49.644235+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":522,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429325+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:42:57.429325+0000","last_peered":"2024-09-13T14:42:57.429325+0000","last_clean":"2024-09-13T14:42:57.429325+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:42:57.429325+0000","last_undegraded":"2024-09-13T14:42:57.429325+0000","last_fullsized":"2024-09-13T14:42:57.429325+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"339'6551","reported_seq":15915,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.984559+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:42:58.984559+0000","last_peered":"2024-09-13T14:42:58.984559+0000","last_clean":"2024-09-13T14:42:58.984559+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:42:58.984559+0000","last_undegraded":"2024-09-13T14:42:58.984559+0000","last_fullsized":"2024-09-13T14:42:58.984559+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315162624,"num_objects":4809,"num_object_clones":0,"num_object_copies":14427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6551,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674640+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:42:49.674640+0000","last_peered":"2024-09-13T14:42:49.674640+0000","last_clean":"2024-09-13T14:42:49.674640+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:42:49.674640+0000","last_undegraded":"2024-09-13T14:42:49.674640+0000","last_fullsized":"2024-09-13T14:42:49.674640+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642796+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:42:49.642796+0000","last_peered":"2024-09-13T14:42:49.642796+0000","last_clean":"2024-09-13T14:42:49.642796+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:42:49.642796+0000","last_undegraded":"2024-09-13T14:42:49.642796+0000","last_fullsized":"2024-09-13T14:42:49.642796+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1101,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429422+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:42:57.429422+0000","last_peered":"2024-09-13T14:42:57.429422+0000","last_clean":"2024-09-13T14:42:57.429422+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:42:57.429422+0000","last_undegraded":"2024-09-13T14:42:57.429422+0000","last_fullsized":"2024-09-13T14:42:57.429422+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":503,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429514+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:42:57.429514+0000","last_peered":"2024-09-13T14:42:57.429514+0000","last_clean":"2024-09-13T14:42:57.429514+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:42:57.429514+0000","last_undegraded":"2024-09-13T14:42:57.429514+0000","last_fullsized":"2024-09-13T14:42:57.429514+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"339'6505","reported_seq":6589,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:43:00.367278+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:43:00.367278+0000","last_peered":"2024-09-13T14:43:00.367278+0000","last_clean":"2024-09-13T14:43:00.367278+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:43:00.367278+0000","last_undegraded":"2024-09-13T14:43:00.367278+0000","last_fullsized":"2024-09-13T14:43:00.367278+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6505,"log_dups_size":0,"ondisk_log_size":6505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307953664,"num_objects":4699,"num_object_clones":0,"num_object_copies":14097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6505,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674618+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:42:49.674618+0000","last_peered":"2024-09-13T14:42:49.674618+0000","last_clean":"2024-09-13T14:42:49.674618+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:42:49.674618+0000","last_undegraded":"2024-09-13T14:42:49.674618+0000","last_fullsized":"2024-09-13T14:42:49.674618+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":512,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429319+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:42:57.429319+0000","last_peered":"2024-09-13T14:42:57.429319+0000","last_clean":"2024-09-13T14:42:57.429319+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:42:57.429319+0000","last_undegraded":"2024-09-13T14:42:57.429319+0000","last_fullsized":"2024-09-13T14:42:57.429319+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":733,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667517+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:42:49.667517+0000","last_peered":"2024-09-13T14:42:49.667517+0000","last_clean":"2024-09-13T14:42:49.667517+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:42:49.667517+0000","last_undegraded":"2024-09-13T14:42:49.667517+0000","last_fullsized":"2024-09-13T14:42:49.667517+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":548,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246392+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:42:57.246392+0000","last_peered":"2024-09-13T14:42:57.246392+0000","last_clean":"2024-09-13T14:42:57.246392+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:42:57.246392+0000","last_undegraded":"2024-09-13T14:42:57.246392+0000","last_fullsized":"2024-09-13T14:42:57.246392+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"339'6789","reported_seq":6873,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:58.521539+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:42:58.521539+0000","last_peered":"2024-09-13T14:42:58.521539+0000","last_clean":"2024-09-13T14:42:58.521539+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:42:58.521539+0000","last_undegraded":"2024-09-13T14:42:58.521539+0000","last_fullsized":"2024-09-13T14:42:58.521539+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6789,"log_dups_size":0,"ondisk_log_size":6789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6789,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644446+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:42:49.644446+0000","last_peered":"2024-09-13T14:42:49.644446+0000","last_clean":"2024-09-13T14:42:49.644446+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:42:49.644446+0000","last_undegraded":"2024-09-13T14:42:49.644446+0000","last_fullsized":"2024-09-13T14:42:49.644446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":548,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.266255+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:42:57.266255+0000","last_peered":"2024-09-13T14:42:57.266255+0000","last_clean":"2024-09-13T14:42:57.266255+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:42:57.266255+0000","last_undegraded":"2024-09-13T14:42:57.266255+0000","last_fullsized":"2024-09-13T14:42:57.266255+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":888,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429682+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:42:57.429682+0000","last_peered":"2024-09-13T14:42:57.429682+0000","last_clean":"2024-09-13T14:42:57.429682+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:42:57.429682+0000","last_undegraded":"2024-09-13T14:42:57.429682+0000","last_fullsized":"2024-09-13T14:42:57.429682+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":546,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.272506+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:42:57.272506+0000","last_peered":"2024-09-13T14:42:57.272506+0000","last_clean":"2024-09-13T14:42:57.272506+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:42:57.272506+0000","last_undegraded":"2024-09-13T14:42:57.272506+0000","last_fullsized":"2024-09-13T14:42:57.272506+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"339'6719","reported_seq":18069,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:43:02.510947+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:43:02.510947+0000","last_peered":"2024-09-13T14:43:02.510947+0000","last_clean":"2024-09-13T14:43:02.510947+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:43:02.510947+0000","last_undegraded":"2024-09-13T14:43:02.510947+0000","last_fullsized":"2024-09-13T14:43:02.510947+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314179608,"num_objects":4795,"num_object_clones":0,"num_object_copies":14385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4795,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6719,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":400,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.267160+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:42:57.267160+0000","last_peered":"2024-09-13T14:42:57.267160+0000","last_clean":"2024-09-13T14:42:57.267160+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:42:57.267160+0000","last_undegraded":"2024-09-13T14:42:57.267160+0000","last_fullsized":"2024-09-13T14:42:57.267160+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":583,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248136+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:42:57.248136+0000","last_peered":"2024-09-13T14:42:57.248136+0000","last_clean":"2024-09-13T14:42:57.248136+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:42:57.248136+0000","last_undegraded":"2024-09-13T14:42:57.248136+0000","last_fullsized":"2024-09-13T14:42:57.248136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":757,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674935+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:42:49.674935+0000","last_peered":"2024-09-13T14:42:49.674935+0000","last_clean":"2024-09-13T14:42:49.674935+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:42:49.674935+0000","last_undegraded":"2024-09-13T14:42:49.674935+0000","last_fullsized":"2024-09-13T14:42:49.674935+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644014+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:42:49.644014+0000","last_peered":"2024-09-13T14:42:49.644014+0000","last_clean":"2024-09-13T14:42:49.644014+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:42:49.644014+0000","last_undegraded":"2024-09-13T14:42:49.644014+0000","last_fullsized":"2024-09-13T14:42:49.644014+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644250+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:42:49.644250+0000","last_peered":"2024-09-13T14:42:49.644250+0000","last_clean":"2024-09-13T14:42:49.644250+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:42:49.644250+0000","last_undegraded":"2024-09-13T14:42:49.644250+0000","last_fullsized":"2024-09-13T14:42:49.644250+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":48,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644110+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:42:49.644110+0000","last_peered":"2024-09-13T14:42:49.644110+0000","last_clean":"2024-09-13T14:42:49.644110+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:42:49.644110+0000","last_undegraded":"2024-09-13T14:42:49.644110+0000","last_fullsized":"2024-09-13T14:42:49.644110+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":500,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429298+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:42:57.429298+0000","last_peered":"2024-09-13T14:42:57.429298+0000","last_clean":"2024-09-13T14:42:57.429298+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:42:57.429298+0000","last_undegraded":"2024-09-13T14:42:57.429298+0000","last_fullsized":"2024-09-13T14:42:57.429298+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":338,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.667430+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:42:49.667430+0000","last_peered":"2024-09-13T14:42:49.667430+0000","last_clean":"2024-09-13T14:42:49.667430+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:42:49.667430+0000","last_undegraded":"2024-09-13T14:42:49.667430+0000","last_fullsized":"2024-09-13T14:42:49.667430+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642839+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:42:49.642839+0000","last_peered":"2024-09-13T14:42:49.642839+0000","last_clean":"2024-09-13T14:42:49.642839+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:42:49.642839+0000","last_undegraded":"2024-09-13T14:42:49.642839+0000","last_fullsized":"2024-09-13T14:42:49.642839+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":505,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.264923+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:42:57.264923+0000","last_peered":"2024-09-13T14:42:57.264923+0000","last_clean":"2024-09-13T14:42:57.264923+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:42:57.264923+0000","last_undegraded":"2024-09-13T14:42:57.264923+0000","last_fullsized":"2024-09-13T14:42:57.264923+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":556,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.245899+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:42:57.245899+0000","last_peered":"2024-09-13T14:42:57.245899+0000","last_clean":"2024-09-13T14:42:57.245899+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:42:57.245899+0000","last_undegraded":"2024-09-13T14:42:57.245899+0000","last_fullsized":"2024-09-13T14:42:57.245899+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":561,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.272454+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:42:57.272454+0000","last_peered":"2024-09-13T14:42:57.272454+0000","last_clean":"2024-09-13T14:42:57.272454+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:42:57.272454+0000","last_undegraded":"2024-09-13T14:42:57.272454+0000","last_fullsized":"2024-09-13T14:42:57.272454+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":336,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.674714+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:42:49.674714+0000","last_peered":"2024-09-13T14:42:49.674714+0000","last_clean":"2024-09-13T14:42:49.674714+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:42:49.674714+0000","last_undegraded":"2024-09-13T14:42:49.674714+0000","last_fullsized":"2024-09-13T14:42:49.674714+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":578,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.272570+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:42:57.272570+0000","last_peered":"2024-09-13T14:42:57.272570+0000","last_clean":"2024-09-13T14:42:57.272570+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:42:57.272570+0000","last_undegraded":"2024-09-13T14:42:57.272570+0000","last_fullsized":"2024-09-13T14:42:57.272570+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":548,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.248343+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:42:57.248343+0000","last_peered":"2024-09-13T14:42:57.248343+0000","last_clean":"2024-09-13T14:42:57.248343+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:42:57.248343+0000","last_undegraded":"2024-09-13T14:42:57.248343+0000","last_fullsized":"2024-09-13T14:42:57.248343+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":525,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429194+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:42:57.429194+0000","last_peered":"2024-09-13T14:42:57.429194+0000","last_clean":"2024-09-13T14:42:57.429194+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:42:57.429194+0000","last_undegraded":"2024-09-13T14:42:57.429194+0000","last_fullsized":"2024-09-13T14:42:57.429194+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":574,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265408+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:42:57.265408+0000","last_peered":"2024-09-13T14:42:57.265408+0000","last_clean":"2024-09-13T14:42:57.265408+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:42:57.265408+0000","last_undegraded":"2024-09-13T14:42:57.265408+0000","last_fullsized":"2024-09-13T14:42:57.265408+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":37,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.655001+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:42:49.655001+0000","last_peered":"2024-09-13T14:42:49.655001+0000","last_clean":"2024-09-13T14:42:49.655001+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:42:49.655001+0000","last_undegraded":"2024-09-13T14:42:49.655001+0000","last_fullsized":"2024-09-13T14:42:49.655001+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":471,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.429521+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:42:57.429521+0000","last_peered":"2024-09-13T14:42:57.429521+0000","last_clean":"2024-09-13T14:42:57.429521+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:42:57.429521+0000","last_undegraded":"2024-09-13T14:42:57.429521+0000","last_fullsized":"2024-09-13T14:42:57.429521+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":552,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.246290+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:42:57.246290+0000","last_peered":"2024-09-13T14:42:57.246290+0000","last_clean":"2024-09-13T14:42:57.246290+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:42:57.246290+0000","last_undegraded":"2024-09-13T14:42:57.246290+0000","last_fullsized":"2024-09-13T14:42:57.246290+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.644184+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:42:49.644184+0000","last_peered":"2024-09-13T14:42:49.644184+0000","last_clean":"2024-09-13T14:42:49.644184+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:42:49.644184+0000","last_undegraded":"2024-09-13T14:42:49.644184+0000","last_fullsized":"2024-09-13T14:42:49.644184+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":304,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642759+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:42:49.642759+0000","last_peered":"2024-09-13T14:42:49.642759+0000","last_clean":"2024-09-13T14:42:49.642759+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:42:49.642759+0000","last_undegraded":"2024-09-13T14:42:49.642759+0000","last_fullsized":"2024-09-13T14:42:49.642759+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.654955+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:42:49.654955+0000","last_peered":"2024-09-13T14:42:49.654955+0000","last_clean":"2024-09-13T14:42:49.654955+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:42:49.654955+0000","last_undegraded":"2024-09-13T14:42:49.654955+0000","last_fullsized":"2024-09-13T14:42:49.654955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":270,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.642988+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:42:49.642988+0000","last_peered":"2024-09-13T14:42:49.642988+0000","last_clean":"2024-09-13T14:42:49.642988+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:42:49.642988+0000","last_undegraded":"2024-09-13T14:42:49.642988+0000","last_fullsized":"2024-09-13T14:42:49.642988+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":570,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265670+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:42:57.265670+0000","last_peered":"2024-09-13T14:42:57.265670+0000","last_clean":"2024-09-13T14:42:57.265670+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:42:57.265670+0000","last_undegraded":"2024-09-13T14:42:57.265670+0000","last_fullsized":"2024-09-13T14:42:57.265670+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":557,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.265823+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:42:57.265823+0000","last_peered":"2024-09-13T14:42:57.265823+0000","last_clean":"2024-09-13T14:42:57.265823+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:42:57.265823+0000","last_undegraded":"2024-09-13T14:42:57.265823+0000","last_fullsized":"2024-09-13T14:42:57.265823+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":338,"state":"active+clean","last_fresh":"2024-09-13T14:42:49.643904+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:42:49.643904+0000","last_peered":"2024-09-13T14:42:49.643904+0000","last_clean":"2024-09-13T14:42:49.643904+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:42:49.643904+0000","last_undegraded":"2024-09-13T14:42:49.643904+0000","last_fullsized":"2024-09-13T14:42:49.643904+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":562,"reported_epoch":339,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.245841+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:42:57.245841+0000","last_peered":"2024-09-13T14:42:57.245841+0000","last_clean":"2024-09-13T14:42:57.245841+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:42:57.245841+0000","last_undegraded":"2024-09-13T14:42:57.245841+0000","last_fullsized":"2024-09-13T14:42:57.245841+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5053480984,"num_objects":77111,"num_object_clones":0,"num_object_copies":231333,"num_objects_missing_on_primary":3801,"num_objects_missing":3801,"num_objects_degraded":40407,"num_objects_misplaced":4755,"num_objects_unfound":0,"num_objects_dirty":77111,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105567,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21524,"num_bytes_recovered":1350172672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13824503808,"data_stored":13824491592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105567,"ondisk_log_size":105567,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397883,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2724076,"kb_used_data":1832228,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91021076,"statfs":{"total":95995035648,"available":93205581824,"internally_reserved":0,"allocated":1876201472,"data_stored":1874448380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594111,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2482968,"kb_used_data":1897232,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91262184,"statfs":{"total":95995035648,"available":93452476416,"internally_reserved":0,"allocated":1942765568,"data_stored":1941032230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665565,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695064,"kb_used_data":358352,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93050088,"statfs":{"total":95995035648,"available":95283290112,"internally_reserved":0,"allocated":366952448,"data_stored":365215441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986566,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969832,"kb_used_data":1074140,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":91775320,"statfs":{"total":95995035648,"available":93977927680,"internally_reserved":0,"allocated":1099919360,"data_stored":1098174366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,2,0,2,2,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697126,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2942164,"kb_used_data":2224268,"kb_used_omap":4,"kb_used_meta":717883,"kb_avail":90802988,"statfs":{"total":95995035648,"available":92982259712,"internally_reserved":0,"allocated":2277650432,"data_stored":2275917578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":735112267},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860653,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2762716,"kb_used_data":2141460,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":90982436,"statfs":{"total":95995035648,"available":93166014464,"internally_reserved":0,"allocated":2192855040,"data_stored":2191114532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020324,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2469136,"kb_used_data":2165576,"kb_used_omap":9,"kb_used_meta":303542,"kb_avail":91276016,"statfs":{"total":95995035648,"available":93466640384,"internally_reserved":0,"allocated":2217549824,"data_stored":2215820443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":310827845},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187714,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490900,"kb_used_data":1867912,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":91254252,"statfs":{"total":95995035648,"available":93444354048,"internally_reserved":0,"allocated":1912741888,"data_stored":1911012773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1903362048,"data_stored":1903362048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2207318016,"data_stored":2207318016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2188251136,"data_stored":2188247064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2268594176,"data_stored":2268594176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1094713344,"data_stored":1094713344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":361959424,"data_stored":361955352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1937113088,"data_stored":1937113088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1863192576,"data_stored":1863188504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:08.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:43:08.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:08.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:08 smithi189 ceph-mon[90992]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 40294/229920 objects degraded (17.525%); 4720/229920 objects misplaced (2.053%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:08.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:08 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40294/229920 objects degraded (17.525%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:08.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:08 smithi017 ceph-mon[103001]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 40294/229920 objects degraded (17.525%); 4720/229920 objects misplaced (2.053%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:08.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:08 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40294/229920 objects degraded (17.525%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:08.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:08 smithi017 ceph-mon[107556]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 40294/229920 objects degraded (17.525%); 4720/229920 objects misplaced (2.053%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:08.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:08 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40294/229920 objects degraded (17.525%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:08.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:08.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.120 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:09.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:09.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:09.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97448 8.2 8:432ac46e:::benchmark_data_smithi017_253885_object6107:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.360 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 85.342921360999995, 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7480373149999999, 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97448 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013454+0000", 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013463+0000", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2749999999999995e-06 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013478+0000", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4827999999999999e-05 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013484+0000", 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0120000000000002e-06 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.732932+0000", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71944847000000001 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.740403+0000", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074705850000000001 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.786063+0000", 2024-09-13T14:43:09.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.045660457000000002 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.794357+0000", 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0082942269999999995 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.642588+0000", 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8482307659999999 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.649051+0000", 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064624849999999996 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.662391+0000", 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01334071 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.705020+0000", 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.042628591 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.736739+0000", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031718728000000002 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.738882+0000", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021435120000000002 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.742772+0000", 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038903879999999998 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.761493+0000", 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01872095 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97452 8.2 8:439122c9:::benchmark_data_smithi017_253885_object6111:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 85.32706254, 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9971937799999999, 2024-09-13T14:43:09.368 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97452 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029313+0000", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029321+0000", 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3140000000000004e-06 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029335+0000", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3889e-05 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029342+0000", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0430000000000002e-06 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.741208+0000", 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.711865953 2024-09-13T14:43:09.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.748863+0000", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076552809999999999 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.862098+0000", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11323446500000001 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.870656+0000", 2024-09-13T14:43:09.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085582220000000007 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.905504+0000", 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0348476689999999 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.912694+0000", 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071900239999999997 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.926156+0000", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013462386 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.999476+0000", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073319630999999996 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.001708+0000", 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022318780000000001 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.004718+0000", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030106780000000001 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.008705+0000", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039868530000000003 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.026509+0000", 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017803531000000001 2024-09-13T14:43:09.375 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97472 8.2 8:4bd78c32:::benchmark_data_smithi017_253885_object6131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 85.185138238999997, 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8783020779999999, 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.376 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97472 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171238+0000", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171245+0000", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6980000000000001e-06 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171256+0000", 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1698e-05 2024-09-13T14:43:09.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171266+0000", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0113e-05 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.871813+0000", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70054635300000001 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.879204+0000", 2024-09-13T14:43:09.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073909620000000001 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.920565+0000", 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.041361235000000003 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.929027+0000", 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084622120000000002 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.969886+0000", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0408586409999998 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.977252+0000", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073662780000000004 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.989408+0000", 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012155454 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.002450+0000", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01304277 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.028897+0000", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.026447114000000001 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.031314+0000", 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024162010000000002 2024-09-13T14:43:09.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.035036+0000", 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037223960000000002 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.049541+0000", 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014505267000000001 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97476 8.2 8:489530ca:::benchmark_data_smithi017_253885_object6135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 85.132599988999999, 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0112176900000001, 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97476 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223776+0000", 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.385 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223785+0000", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223799+0000", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4086e-05 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223807+0000", 2024-09-13T14:43:09.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1669999999999999e-06 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.883412+0000", 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.65960507599999996 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.890980+0000", 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075679470000000002 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.959455+0000", 2024-09-13T14:43:09.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068474876000000004 2024-09-13T14:43:09.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.968466+0000", 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0090107199999999998 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.105941+0000", 2024-09-13T14:43:09.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.137475292 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.113301+0000", 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073604889999999996 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.125657+0000", 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012355319999999999 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.162775+0000", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.037118233 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.176717+0000", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013941808 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.216950+0000", 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040233172999999997 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.220608+0000", 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036583179999999998 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.234995+0000", 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01438681 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.392 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98595 8.2 8:4695e799:::benchmark_data_smithi017_253885_object7254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 73.115959515, 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.745307597, 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98595 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240416+0000", 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240425+0000", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0589999999999992e-06 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240440+0000", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4465e-05 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240447+0000", 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7150000000000001e-06 2024-09-13T14:43:09.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.413119+0000", 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.172672935 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.420808+0000", 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076884919999999999 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.489162+0000", 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068353763999999997 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.497299+0000", 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081373120000000007 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.875937+0000", 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3786378049999999 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.884179+0000", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0082421079999999997 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.901112+0000", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016932966000000001 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.954696+0000", 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.053583700999999997 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.959435+0000", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047390230000000002 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.961626+0000", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00219162 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.965435+0000", 2024-09-13T14:43:09.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038089930000000001 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.985726+0000", 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020290269 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98605 8.2 8:41a3b6c0:::benchmark_data_smithi017_253885_object7264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 72.902267058000007, 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7582236789999999, 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98605 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454109+0000", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454115+0000", 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9090000000000003e-06 2024-09-13T14:43:09.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454128+0000", 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2670999999999999e-05 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454135+0000", 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.776e-06 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.403 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.498066+0000", 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.043930780000000003 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.507224+0000", 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091585580000000007 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.553871+0000", 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046647010000000003 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.563511+0000", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0096394859999999992 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.090575+0000", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5270640310000001 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.097314+0000", 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006738979 2024-09-13T14:43:09.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.109230+0000", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011916456000000001 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.184619+0000", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075389251000000004 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.187198+0000", 2024-09-13T14:43:09.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025785669999999999 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.191458+0000", 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004260107 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.195445+0000", 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039864619999999996 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.212334+0000", 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016889441000000002 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98654 8.2 8:4498e4e3:::benchmark_data_smithi017_253885_object7313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 72.368966020000002, 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7594836640000002, 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.408 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98654 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987410+0000", 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987416+0000", 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9050000000000002e-06 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.410 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987427+0000", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1325000000000001e-05 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987432+0000", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7650000000000001e-06 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.068663+0000", 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.081231138999999994 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.076300+0000", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076368790000000001 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.143490+0000", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.067189789 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.152408+0000", 2024-09-13T14:43:09.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.008918367 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.615689+0000", 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4632809 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.623737+0000", 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080478649999999995 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.639371+0000", 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015633463 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.722883+0000", 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083512102000000005 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.725216+0000", 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002332804 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.728339+0000", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031238519999999999 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.732036+0000", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003697035 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.746895+0000", 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014858675999999999 2024-09-13T14:43:09.415 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98659 8.2 8:4652443c:::benchmark_data_smithi017_253885_object7318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 72.086650129000006, 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8136869770000001, 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.416 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98659 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269726+0000", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269732+0000", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.959e-06 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269744+0000", 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1746000000000001e-05 2024-09-13T14:43:09.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269750+0000", 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8560000000000003e-06 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.292615+0000", 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022865963999999999 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.300310+0000", 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076944789999999997 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.496894+0000", 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19658410300000001 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.505348+0000", 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084539760000000002 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.964988+0000", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.459640442 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.971768+0000", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0067792420000000004 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.983698+0000", 2024-09-13T14:43:09.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011930690000000001 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.058423+0000", 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.074724156 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.060669+0000", 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022467469999999999 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.063945+0000", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032759719999999998 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.067424+0000", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003479102 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.083414+0000", 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015989894000000001 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.423 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98859 8.2 8:440b55f0:::benchmark_data_smithi017_253885_object7518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 70.789741575999997, 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8518238469999999, 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98859 2024-09-13T14:43:09.424 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.425 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566634+0000", 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566642+0000", 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8890000000000002e-06 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566654+0000", 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1341e-05 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.426 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566659+0000", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8570000000000003e-06 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.732707+0000", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.16604880599999999 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.741658+0000", 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0089508469999999996 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.812527+0000", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.070868908999999994 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.827305+0000", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014777521 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.320283+0000", 2024-09-13T14:43:09.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4929778950000001 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.327304+0000", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070219059999999996 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.339435+0000", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012130448 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.383514+0000", 2024-09-13T14:43:09.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.044078719000000002 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.387513+0000", 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039998100000000003 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.391087+0000", 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035737669999999998 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.397274+0000", 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061866619999999999 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.418460+0000", 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021185874 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98909 8.2 8:4a9c011b:::benchmark_data_smithi017_253885_object7568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.431 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 70.326653187000005, 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0264528579999999, 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98909 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:43:09.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029723+0000", 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029730+0000", 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0090000000000001e-06 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029743+0000", 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3865e-05 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029751+0000", 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4429999999999997e-06 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.216354+0000", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18660349200000001 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.226081+0000", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0097264250000000003 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.295483+0000", 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.069402162000000003 2024-09-13T14:43:09.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.309279+0000", 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013796294000000001 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.921663+0000", 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6123836869999999 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.928931+0000", 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072678009999999999 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.946660+0000", 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017729041000000001 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.982389+0000", 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035728754000000001 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.988132+0000", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057438530000000002 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.026695+0000", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.038562415000000003 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.031498+0000", 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004802906 2024-09-13T14:43:09.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.056177+0000", 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024679390999999998 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98993 8.2 8:46d18c53:::benchmark_data_smithi017_253885_object7652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 69.586405102000001, 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8958036410000001, 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.439 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98993 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769970+0000", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769978+0000", 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.525e-06 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769990+0000", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2349e-05 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769995+0000", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1240000000000004e-06 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.864919+0000", 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094923341999999994 2024-09-13T14:43:09.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.873053+0000", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081339970000000004 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.920149+0000", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047096617 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.930004+0000", 2024-09-13T14:43:09.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0098543599999999995 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.599878+0000", 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6698743540000001 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.607075+0000", 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071964400000000001 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.619997+0000", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012922703000000001 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.638895+0000", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018897450999999999 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.639635+0000", 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00074067800000000004 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.642118+0000", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024828799999999998 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.645888+0000", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037692960000000001 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.665776+0000", 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019888385000000001 2024-09-13T14:43:09.446 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99143 8.2 8:4f3b130b:::benchmark_data_smithi017_253885_object7802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.719271442999997, 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7602121840000002, 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.447 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99143 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637104+0000", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637113+0000", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4479999999999994e-06 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637128+0000", 2024-09-13T14:43:09.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5591e-05 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637136+0000", 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9789999999999993e-06 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.842593+0000", 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20545687000000001 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.852984+0000", 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010390893 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.003182+0000", 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.150197414 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.012316+0000", 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.009134171 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.221151+0000", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2088349360000001 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.227828+0000", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066769289999999999 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.239426+0000", 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011598308 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.311405+0000", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.071979498000000003 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.341966+0000", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030560744000000001 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.373730+0000", 2024-09-13T14:43:09.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031763668000000002 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.377836+0000", 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041062160000000002 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.397318+0000", 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019482152999999999 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99146 8.2 8:499fc9a0:::benchmark_data_smithi017_253885_object7805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.699484178000006, 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7721449680000001, 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99146 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656892+0000", 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656899+0000", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1189999999999999e-06 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656912+0000", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2971e-05 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656918+0000", 2024-09-13T14:43:09.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.013329+0000", 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.35641155200000002 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.022916+0000", 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095863830000000004 2024-09-13T14:43:09.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:09.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.075239+0000", 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052323242999999998 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.091427+0000", 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016188428000000001 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.279679+0000", 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1882518329999998 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.287334+0000", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076550480000000002 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.299647+0000", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012312963999999999 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.398525+0000", 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.098877815999999993 2024-09-13T14:43:09.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.401134+0000", 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026091700000000001 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.404614+0000", 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034802000000000001 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.463 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.408343+0000", 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037289850000000002 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.429038+0000", 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020694791000000001 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99172 8.2 8:4a96f2ce:::benchmark_data_smithi017_253885_object7831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:43:09.464 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 68.423530732000003, 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8392140829999999, 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99172 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:43:09.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932844+0000", 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932854+0000", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2089999999999994e-06 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932868+0000", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4715000000000001e-05 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932878+0000", 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1900000000000001e-06 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.137866+0000", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20498866900000001 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.145900+0000", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080335430000000006 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.197956+0000", 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052055782000000002 2024-09-13T14:43:09.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.206529+0000", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085734160000000004 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.624724+0000", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.418195297 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.632205+0000", 2024-09-13T14:43:09.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074804520000000003 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.646855+0000", 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014649970999999999 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.744121+0000", 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097266562000000001 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.746620+0000", 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002498648 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.748823+0000", 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002202673 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.752588+0000", 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003765876 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.772061+0000", 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019472322 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99178 8.2 8:45566d88:::benchmark_data_smithi017_253885_object7837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 67.960873333999999, 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.922370785, 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.472 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99178 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395502+0000", 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395510+0000", 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9359999999999999e-06 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395525+0000", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4917e-05 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395532+0000", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9909999999999996e-06 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.426417+0000", 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030884630999999999 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.434362+0000", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0079452840000000004 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.628865+0000", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19450308499999999 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.639250+0000", 2024-09-13T14:43:09.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010384581 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.139981+0000", 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5007314940000001 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.147758+0000", 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077768819999999997 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.160527+0000", 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012769019 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.235879+0000", 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075351561999999997 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.295287+0000", 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.059408395000000003 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.299011+0000", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037238200000000001 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.303047+0000", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040359360000000004 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.317875+0000", 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014828182000000001 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.479 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99338 8.2 8:41ebf48b:::benchmark_data_smithi017_253885_object7997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 66.418462564999999, 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8585652349999999, 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.480 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99338 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.481 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937913+0000", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937921+0000", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9990000000000008e-06 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937936+0000", 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4851999999999999e-05 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937942+0000", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2129999999999996e-06 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.971043+0000", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.033100282000000002 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.979114+0000", 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080716690000000001 2024-09-13T14:43:09.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.126251+0000", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14713672799999999 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.134698+0000", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084468640000000001 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.611327+0000", 2024-09-13T14:43:09.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4766293250000002 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.622937+0000", 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011609647000000001 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.635618+0000", 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012681143000000001 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.708910+0000", 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073292369999999996 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.709651+0000", 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00074075200000000001 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.775615+0000", 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.065963663000000006 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.779356+0000", 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037407 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.796480+0000", 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017124550999999998 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99635 8.2 8:4844b8d9:::benchmark_data_smithi017_253885_object8294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.487 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 64.097954154000007, 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7339699610000001, 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99635 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258422+0000", 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258428+0000", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0569999999999998e-06 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258437+0000", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1039999999999996e-06 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258442+0000", 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2359999999999998e-06 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.441876+0000", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18343456899999999 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.450359+0000", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084833749999999996 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.532946+0000", 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082586406000000001 2024-09-13T14:43:09.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.543896+0000", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010949898 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.827699+0000", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2838030069999999 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.835173+0000", 2024-09-13T14:43:09.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074745250000000001 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.847362+0000", 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012188516 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.926601+0000", 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079239233000000006 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.927360+0000", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00075908799999999999 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.968013+0000", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040653214 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.971873+0000", 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003859974 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.992393+0000", 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020519936999999999 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100002 8.2 8:4bf97488:::benchmark_data_smithi017_253885_object8661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 60.125783640000002, 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8636257819999997, 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.495 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100002 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.496 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230593+0000", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230599+0000", 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1630000000000001e-06 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230608+0000", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5680000000000005e-06 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230613+0000", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.622e-06 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.288032+0000", 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.057418652000000001 2024-09-13T14:43:09.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.298634+0000", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010602504 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.412808+0000", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.114174079 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.421456+0000", 2024-09-13T14:43:09.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0086479599999999997 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.995933+0000", 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5744767720000001 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.002777+0000", 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068444159999999999 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.014676+0000", 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011898283000000001 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.056025+0000", 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.041349823000000001 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.059989+0000", 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039637860000000004 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.061952+0000", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019627899999999998 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.069233+0000", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072812229999999999 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.094220+0000", 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024986313 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.502 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100010 8.2 8:496ef423:::benchmark_data_smithi017_253885_object8669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 60.048739849, 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.744854245, 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100010 2024-09-13T14:43:09.503 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307636+0000", 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307643+0000", 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8909999999999998e-06 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307656+0000", 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2782000000000001e-05 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307662+0000", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9039999999999997e-06 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.454671+0000", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14700880499999999 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.462987+0000", 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0083164420000000003 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.535549+0000", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.072561821999999998 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.544006+0000", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084573340000000004 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.963417+0000", 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4194113389999998 2024-09-13T14:43:09.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.970485+0000", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070679580000000001 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.983275+0000", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012789327 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.026413+0000", 2024-09-13T14:43:09.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.043137847999999999 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.029413+0000", 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030000489999999999 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.030380+0000", 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00096762300000000003 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.034053+0000", 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036728160000000002 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.052492+0000", 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018438754000000002 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100020 8.2 8:480f1fee:::benchmark_data_smithi017_253885_object8679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:43:09.510 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 59.835374897000001, 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.817903598, 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100020 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521001+0000", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521009+0000", 2024-09-13T14:43:09.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3149999999999992e-06 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521023+0000", 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4157e-05 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521031+0000", 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3989999999999998e-06 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.544782+0000", 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023750996 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.552261+0000", 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074796719999999997 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.598595+0000", 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046333107999999998 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.607874+0000", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0092794469999999997 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.170772+0000", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.562898235 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.180804+0000", 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010031833 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.200140+0000", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019335871000000001 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.254006+0000", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.053865937000000003 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.267136+0000", 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013130372 2024-09-13T14:43:09.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.316011+0000", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048874362999999997 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.320353+0000", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043422640000000002 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.338906+0000", 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018553284 2024-09-13T14:43:09.517 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.518 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.518 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.518 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:09.518 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:09.518 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:43:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:09.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:09.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:09.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:09.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:09.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:09.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:09.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:09.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:43:09.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:10.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:10.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:10.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:10.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:10.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:10.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:43:10.363 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:10.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:10.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:10.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:10.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:10.640 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:10.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:10.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:10.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:10.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:10.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:10.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:10.926 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:10 smithi189 ceph-mon[90992]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40260/229308 objects degraded (17.557%); 4684/229308 objects misplaced (2.043%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:10 smithi017 ceph-mon[103001]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40260/229308 objects degraded (17.557%); 4684/229308 objects misplaced (2.043%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:10 smithi017 ceph-mon[107556]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40260/229308 objects degraded (17.557%); 4684/229308 objects misplaced (2.043%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:10.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:10.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:11.170 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:11.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:11.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:11.589 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:11.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 40260/228573 objects degraded (17.614%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1389,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":340,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":76191,"data_bytes":4996041398,"bytes_used":18846027776,"bytes_avail":749114257408,"bytes_total":767960285184,"degraded_objects":40260,"degraded_total":228573,"degraded_ratio":0.17613628906301268,"misplaced_objects":4684,"misplaced_total":228573,"misplaced_ratio":0.020492359115031084,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":969678,"recovering_keys_per_sec":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":111},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 18s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:43:11.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:11.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:11.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:12.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:12.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:12.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:12.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:12.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:12.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:12.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:12.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:12.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:12.443 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:43:12.443 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:43:12.443 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:43:12.443 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:43:12.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:12.561 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:12.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:12.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:12 smithi189 ceph-mon[90992]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 40260/228573 objects degraded (17.614%); 4684/228573 objects misplaced (2.049%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1860531129' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:12.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:12.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:12.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:12.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[103001]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 40260/228573 objects degraded (17.614%); 4684/228573 objects misplaced (2.049%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1860531129' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[107556]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 40260/228573 objects degraded (17.614%); 4684/228573 objects misplaced (2.049%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1860531129' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:13.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:13.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:13.144 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:43:13.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:43:13.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:43:13.168 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:13.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:13.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:13.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:13.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:13.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:13.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:13.474 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:13.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:13.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:13.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:13.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:13.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:13.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:13.902 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:13 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40260/228573 objects degraded (17.614%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:13.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:13.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:13 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40260/228573 objects degraded (17.614%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:13 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40260/228573 objects degraded (17.614%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:14.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:14.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:14.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:14.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:14.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:14.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:14.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:14.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:14.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:14.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:14.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:14.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:14.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:14.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:14 smithi017 ceph-mon[103001]: pgmap v1012: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 40208/228099 objects degraded (17.627%); 4684/228099 objects misplaced (2.053%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:14 smithi017 ceph-mon[107556]: pgmap v1012: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 40208/228099 objects degraded (17.627%); 4684/228099 objects misplaced (2.053%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:15.003 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:14 smithi189 ceph-mon[90992]: pgmap v1012: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 40208/228099 objects degraded (17.627%); 4684/228099 objects misplaced (2.053%); 870 KiB/s, 13 objects/s recovering 2024-09-13T14:43:15.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:15.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:15.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:15.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:15.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:15.254 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:43:15.255 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:43:15.255 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:43:15.255 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:43:15.255 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:43:15.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:15.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:15.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:15.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:15.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:15.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:15.577 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:15.679 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:15.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:15.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:15.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:15.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:15.942 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:43:15.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:16.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:16.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:16.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:16.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:16.232 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:16.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:16.330 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:16.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:16.456 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:16.558 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:16.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:16.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:16.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:16.599 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:16.599 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:16.617 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:16.617 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:16.635 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:16.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:16.653 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:16.654 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:16.671 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:16.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:16.680 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:43:16.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:16.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:16.696 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:16.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:16.720 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:16.721 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:16.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[103001]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 40175/227043 objects degraded (17.695%); 4651/227043 objects misplaced (2.049%); 857 KiB/s, 13 objects/s recovering 2024-09-13T14:43:16.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:16.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2935003429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:16.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[107556]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 40175/227043 objects degraded (17.695%); 4651/227043 objects misplaced (2.049%); 857 KiB/s, 13 objects/s recovering 2024-09-13T14:43:16.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:16.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2935003429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:16.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:16.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:16.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:16.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:16 smithi189 ceph-mon[90992]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 40175/227043 objects degraded (17.695%); 4651/227043 objects misplaced (2.049%); 857 KiB/s, 13 objects/s recovering 2024-09-13T14:43:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2935003429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:17.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:17.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:17.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:17.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:17.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:17.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:17] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.51.0" 2024-09-13T14:43:17.456 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:17.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:17.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:17.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:17.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:17 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:17.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:17.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:17 smithi189 ceph-mon[90992]: osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:43:17.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:17.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:17.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[103001]: osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2072095241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:17 smithi017 ceph-mon[107556]: osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:43:17.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:18.049 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:18.049 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:18.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:18.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:18.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:18.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:18.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:18.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:18.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:18.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:18.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:18.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:18.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:43:18.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:18.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:18.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:18 smithi017 ceph-mon[103001]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40059/226296 objects degraded (17.702%); 4651/226296 objects misplaced (2.055%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:43:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:18 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40059/226296 objects degraded (17.702%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:18 smithi017 ceph-mon[107556]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40059/226296 objects degraded (17.702%); 4651/226296 objects misplaced (2.055%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:43:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:18 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40059/226296 objects degraded (17.702%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:18.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:18 smithi189 ceph-mon[90992]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 40059/226296 objects degraded (17.702%); 4651/226296 objects misplaced (2.055%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:43:19.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:18 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40059/226296 objects degraded (17.702%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:19.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:19.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:19.090 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:19.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:19.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:19.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:19.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:19.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:19.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:19.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:19.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:20.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:20.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:20.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:20.219 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:20.220 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:43:20.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:20.325 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:20.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:20.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:20.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:20.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:20.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:20.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:20.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:20.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:20.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:20.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:20.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:20.896 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:20.896 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:20.905 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:20 smithi189 ceph-mon[90992]: pgmap v1016: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 40025/225564 objects degraded (17.744%); 4617/225564 objects misplaced (2.047%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:20 smithi017 ceph-mon[103001]: pgmap v1016: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 40025/225564 objects degraded (17.744%); 4617/225564 objects misplaced (2.047%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:20 smithi017 ceph-mon[107556]: pgmap v1016: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 40025/225564 objects degraded (17.744%); 4617/225564 objects misplaced (2.047%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:20.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:20.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:21.057 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1016,"stamp":"2024-09-13T14:43:19.187526+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4930308790,"num_objects":75188,"num_object_clones":0,"num_object_copies":225564,"num_objects_missing_on_primary":3665,"num_objects_missing":3665,"num_objects_degraded":40025,"num_objects_misplaced":4617,"num_objects_unfound":0,"num_objects_dirty":75188,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":108465,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21842,"num_bytes_recovered":1367443613,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109650,"ondisk_log_size":109650,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18195524,"kb_used_data":13223984,"kb_used_omap":44,"kb_used_meta":4971411,"kb_avail":731765692,"statfs":{"total":767960285184,"available":749328068608,"internally_reserved":0,"allocated":13541359616,"data_stored":13527349469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5090725111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,5,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81788928,"num_objects":-1248,"num_object_clones":0,"num_object_copies":-3744,"num_objects_missing_on_primary":-67,"num_objects_missing":-67,"num_objects_degraded":-235,"num_objects_misplaced":-67,"num_objects_unfound":0,"num_objects_dirty":-1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1248,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":9895936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002673"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":563,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.664985+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:43:06.664985+0000","last_peered":"2024-09-13T14:43:06.664985+0000","last_clean":"2024-09-13T14:43:06.664985+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:43:06.664985+0000","last_undegraded":"2024-09-13T14:43:06.664985+0000","last_fullsized":"2024-09-13T14:43:06.664985+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":339,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673833+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:43:06.673833+0000","last_peered":"2024-09-13T14:43:06.673833+0000","last_clean":"2024-09-13T14:43:06.673833+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:43:06.673833+0000","last_undegraded":"2024-09-13T14:43:06.673833+0000","last_fullsized":"2024-09-13T14:43:06.673833+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657404+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:43:06.657404+0000","last_peered":"2024-09-13T14:43:06.657404+0000","last_clean":"2024-09-13T14:43:06.657404+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:43:06.657404+0000","last_undegraded":"2024-09-13T14:43:06.657404+0000","last_fullsized":"2024-09-13T14:43:06.657404+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":550,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733936+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:43:16.733936+0000","last_peered":"2024-09-13T14:43:16.733936+0000","last_clean":"2024-09-13T14:43:16.733936+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:43:16.733936+0000","last_undegraded":"2024-09-13T14:43:16.733936+0000","last_fullsized":"2024-09-13T14:43:16.733936+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":550,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734260+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:43:16.734260+0000","last_peered":"2024-09-13T14:43:16.734260+0000","last_clean":"2024-09-13T14:43:16.734260+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:43:16.734260+0000","last_undegraded":"2024-09-13T14:43:16.734260+0000","last_fullsized":"2024-09-13T14:43:16.734260+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":308,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673669+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:43:06.673669+0000","last_peered":"2024-09-13T14:43:06.673669+0000","last_clean":"2024-09-13T14:43:06.673669+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:43:06.673669+0000","last_undegraded":"2024-09-13T14:43:06.673669+0000","last_fullsized":"2024-09-13T14:43:06.673669+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":562,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.665716+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:43:06.665716+0000","last_peered":"2024-09-13T14:43:06.665716+0000","last_clean":"2024-09-13T14:43:06.665716+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:43:06.665716+0000","last_undegraded":"2024-09-13T14:43:06.665716+0000","last_fullsized":"2024-09-13T14:43:06.665716+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":330,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816650+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:43:06.816650+0000","last_peered":"2024-09-13T14:43:06.816650+0000","last_clean":"2024-09-13T14:43:06.816650+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:43:06.816650+0000","last_undegraded":"2024-09-13T14:43:06.816650+0000","last_fullsized":"2024-09-13T14:43:06.816650+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"341'6647","reported_seq":9735,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:16.927113+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:43:16.927113+0000","last_peered":"2024-09-13T14:43:16.927113+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:43:16.927113+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6647,"log_dups_size":0,"ondisk_log_size":6647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304283648,"num_objects":4643,"num_object_clones":0,"num_object_copies":13929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8201,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6647,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1002,"num_bytes_recovered":65667072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3659}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":334,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674328+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:43:06.674328+0000","last_peered":"2024-09-13T14:43:06.674328+0000","last_clean":"2024-09-13T14:43:06.674328+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:43:06.674328+0000","last_undegraded":"2024-09-13T14:43:06.674328+0000","last_fullsized":"2024-09-13T14:43:06.674328+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657054+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:43:06.657054+0000","last_peered":"2024-09-13T14:43:06.657054+0000","last_clean":"2024-09-13T14:43:06.657054+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:43:06.657054+0000","last_undegraded":"2024-09-13T14:43:06.657054+0000","last_fullsized":"2024-09-13T14:43:06.657054+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"341'6747","reported_seq":16285,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:17.917164+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:43:17.917164+0000","last_peered":"2024-09-13T14:43:17.917164+0000","last_clean":"2024-09-13T14:43:17.917164+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:43:17.917164+0000","last_undegraded":"2024-09-13T14:43:17.917164+0000","last_fullsized":"2024-09-13T14:43:17.917164+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310706176,"num_objects":4741,"num_object_clones":0,"num_object_copies":14223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6747,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674402+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:43:06.674402+0000","last_peered":"2024-09-13T14:43:06.674402+0000","last_clean":"2024-09-13T14:43:06.674402+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:43:06.674402+0000","last_undegraded":"2024-09-13T14:43:06.674402+0000","last_fullsized":"2024-09-13T14:43:06.674402+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739721+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:43:06.739721+0000","last_peered":"2024-09-13T14:43:06.739721+0000","last_clean":"2024-09-13T14:43:06.739721+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:43:06.739721+0000","last_undegraded":"2024-09-13T14:43:06.739721+0000","last_fullsized":"2024-09-13T14:43:06.739721+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"341'6979","reported_seq":7067,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:18.036529+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:43:18.036529+0000","last_peered":"2024-09-13T14:43:18.036529+0000","last_clean":"2024-09-13T14:43:18.036529+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:43:18.036529+0000","last_undegraded":"2024-09-13T14:43:18.036529+0000","last_fullsized":"2024-09-13T14:43:18.036529+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315817984,"num_objects":4819,"num_object_clones":0,"num_object_copies":14457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6979,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674296+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:43:06.674296+0000","last_peered":"2024-09-13T14:43:06.674296+0000","last_clean":"2024-09-13T14:43:06.674296+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:43:06.674296+0000","last_undegraded":"2024-09-13T14:43:06.674296+0000","last_fullsized":"2024-09-13T14:43:06.674296+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":321,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816842+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:43:06.816842+0000","last_peered":"2024-09-13T14:43:06.816842+0000","last_clean":"2024-09-13T14:43:06.816842+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:43:06.816842+0000","last_undegraded":"2024-09-13T14:43:06.816842+0000","last_fullsized":"2024-09-13T14:43:06.816842+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"340'6794","reported_seq":6880,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:14.661698+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:43:14.661698+0000","last_peered":"2024-09-13T14:43:14.661698+0000","last_clean":"2024-09-13T14:43:14.661698+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:43:14.661698+0000","last_undegraded":"2024-09-13T14:43:14.661698+0000","last_fullsized":"2024-09-13T14:43:14.661698+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6794,"log_dups_size":0,"ondisk_log_size":6794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311427072,"num_objects":4752,"num_object_clones":0,"num_object_copies":14256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6794,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":269,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657276+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:43:06.657276+0000","last_peered":"2024-09-13T14:43:06.657276+0000","last_clean":"2024-09-13T14:43:06.657276+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:43:06.657276+0000","last_undegraded":"2024-09-13T14:43:06.657276+0000","last_fullsized":"2024-09-13T14:43:06.657276+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":576,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733921+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:43:16.733921+0000","last_peered":"2024-09-13T14:43:16.733921+0000","last_clean":"2024-09-13T14:43:16.733921+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:43:16.733921+0000","last_undegraded":"2024-09-13T14:43:16.733921+0000","last_fullsized":"2024-09-13T14:43:16.733921+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"340'6749","reported_seq":6832,"reported_epoch":340,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:15.602055+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:43:15.602055+0000","last_peered":"2024-09-13T14:43:15.602055+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:43:15.602055+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308871168,"num_objects":4713,"num_object_clones":0,"num_object_copies":14139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6749,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-920}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":452,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673739+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:43:06.673739+0000","last_peered":"2024-09-13T14:43:06.673739+0000","last_clean":"2024-09-13T14:43:06.673739+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:43:06.673739+0000","last_undegraded":"2024-09-13T14:43:06.673739+0000","last_fullsized":"2024-09-13T14:43:06.673739+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":572,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.665177+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:43:06.665177+0000","last_peered":"2024-09-13T14:43:06.665177+0000","last_clean":"2024-09-13T14:43:06.665177+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:43:06.665177+0000","last_undegraded":"2024-09-13T14:43:06.665177+0000","last_fullsized":"2024-09-13T14:43:06.665177+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739916+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:43:06.739916+0000","last_peered":"2024-09-13T14:43:06.739916+0000","last_clean":"2024-09-13T14:43:06.739916+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:43:06.739916+0000","last_undegraded":"2024-09-13T14:43:06.739916+0000","last_fullsized":"2024-09-13T14:43:06.739916+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"340'6475","reported_seq":6561,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:14.564681+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:43:14.564681+0000","last_peered":"2024-09-13T14:43:14.564681+0000","last_clean":"2024-09-13T14:43:14.564681+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:43:14.564681+0000","last_undegraded":"2024-09-13T14:43:14.564681+0000","last_fullsized":"2024-09-13T14:43:14.564681+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":156,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656816+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:43:06.656816+0000","last_peered":"2024-09-13T14:43:06.656816+0000","last_clean":"2024-09-13T14:43:06.656816+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:43:06.656816+0000","last_undegraded":"2024-09-13T14:43:06.656816+0000","last_fullsized":"2024-09-13T14:43:06.656816+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":342,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816739+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:43:06.816739+0000","last_peered":"2024-09-13T14:43:06.816739+0000","last_clean":"2024-09-13T14:43:06.816739+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:43:06.816739+0000","last_undegraded":"2024-09-13T14:43:06.816739+0000","last_fullsized":"2024-09-13T14:43:06.816739+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":54,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740127+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:43:06.740127+0000","last_peered":"2024-09-13T14:43:06.740127+0000","last_clean":"2024-09-13T14:43:06.740127+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:43:06.740127+0000","last_undegraded":"2024-09-13T14:43:06.740127+0000","last_fullsized":"2024-09-13T14:43:06.740127+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"341'6692","reported_seq":9350,"reported_epoch":341,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:18.417299+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:43:18.417299+0000","last_peered":"2024-09-13T14:43:18.417299+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:43:18.417299+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":3665,"num_objects_missing":3665,"num_objects_degraded":4040,"num_objects_misplaced":4617,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6692,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":986,"num_bytes_recovered":64618496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3665},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739579+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:43:06.739579+0000","last_peered":"2024-09-13T14:43:06.739579+0000","last_clean":"2024-09-13T14:43:06.739579+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:43:06.739579+0000","last_undegraded":"2024-09-13T14:43:06.739579+0000","last_fullsized":"2024-09-13T14:43:06.739579+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740150+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:43:06.740150+0000","last_peered":"2024-09-13T14:43:06.740150+0000","last_clean":"2024-09-13T14:43:06.740150+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:43:06.740150+0000","last_undegraded":"2024-09-13T14:43:06.740150+0000","last_fullsized":"2024-09-13T14:43:06.740150+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"340'6561","reported_seq":6647,"reported_epoch":340,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:14.658107+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:43:14.658107+0000","last_peered":"2024-09-13T14:43:14.658107+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:43:14.658107+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-869}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":149,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657157+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:43:06.657157+0000","last_peered":"2024-09-13T14:43:06.657157+0000","last_clean":"2024-09-13T14:43:06.657157+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:43:06.657157+0000","last_undegraded":"2024-09-13T14:43:06.657157+0000","last_fullsized":"2024-09-13T14:43:06.657157+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":564,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.672143+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:43:06.672143+0000","last_peered":"2024-09-13T14:43:06.672143+0000","last_clean":"2024-09-13T14:43:06.672143+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:43:06.672143+0000","last_undegraded":"2024-09-13T14:43:06.672143+0000","last_fullsized":"2024-09-13T14:43:06.672143+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":526,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658784+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:43:06.658784+0000","last_peered":"2024-09-13T14:43:06.658784+0000","last_clean":"2024-09-13T14:43:06.658784+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:43:06.658784+0000","last_undegraded":"2024-09-13T14:43:06.658784+0000","last_fullsized":"2024-09-13T14:43:06.658784+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"341'6613","reported_seq":6666,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:17.973903+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:43:17.973903+0000","last_peered":"2024-09-13T14:43:17.973903+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:43:17.973903+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6613,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-889}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":443,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673826+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:43:06.673826+0000","last_peered":"2024-09-13T14:43:06.673826+0000","last_clean":"2024-09-13T14:43:06.673826+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:43:06.673826+0000","last_undegraded":"2024-09-13T14:43:06.673826+0000","last_fullsized":"2024-09-13T14:43:06.673826+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":865,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656967+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:43:06.656967+0000","last_peered":"2024-09-13T14:43:06.656967+0000","last_clean":"2024-09-13T14:43:06.656967+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:43:06.656967+0000","last_undegraded":"2024-09-13T14:43:06.656967+0000","last_fullsized":"2024-09-13T14:43:06.656967+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":518,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.664538+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:43:06.664538+0000","last_peered":"2024-09-13T14:43:06.664538+0000","last_clean":"2024-09-13T14:43:06.664538+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:43:06.664538+0000","last_undegraded":"2024-09-13T14:43:06.664538+0000","last_fullsized":"2024-09-13T14:43:06.664538+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.817173+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:43:06.817173+0000","last_peered":"2024-09-13T14:43:06.817173+0000","last_clean":"2024-09-13T14:43:06.817173+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:43:06.817173+0000","last_undegraded":"2024-09-13T14:43:06.817173+0000","last_fullsized":"2024-09-13T14:43:06.817173+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"340'6784","reported_seq":6868,"reported_epoch":340,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:14.412108+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:43:14.412108+0000","last_peered":"2024-09-13T14:43:14.412108+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:43:14.412108+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6784,"log_dups_size":0,"ondisk_log_size":6784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312213504,"num_objects":4764,"num_object_clones":0,"num_object_copies":14292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6784,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-916}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":372,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.659077+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:43:06.659077+0000","last_peered":"2024-09-13T14:43:06.659077+0000","last_clean":"2024-09-13T14:43:06.659077+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:43:06.659077+0000","last_undegraded":"2024-09-13T14:43:06.659077+0000","last_fullsized":"2024-09-13T14:43:06.659077+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":731,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673750+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:43:06.673750+0000","last_peered":"2024-09-13T14:43:06.673750+0000","last_clean":"2024-09-13T14:43:06.673750+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:43:06.673750+0000","last_undegraded":"2024-09-13T14:43:06.673750+0000","last_fullsized":"2024-09-13T14:43:06.673750+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":569,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733841+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:43:16.733841+0000","last_peered":"2024-09-13T14:43:16.733841+0000","last_clean":"2024-09-13T14:43:16.733841+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:43:16.733841+0000","last_undegraded":"2024-09-13T14:43:16.733841+0000","last_fullsized":"2024-09-13T14:43:16.733841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":574,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734371+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:43:16.734371+0000","last_peered":"2024-09-13T14:43:16.734371+0000","last_clean":"2024-09-13T14:43:16.734371+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:43:16.734371+0000","last_undegraded":"2024-09-13T14:43:16.734371+0000","last_fullsized":"2024-09-13T14:43:16.734371+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"340'6824","reported_seq":16451,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:15.398165+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:43:15.398165+0000","last_peered":"2024-09-13T14:43:15.398165+0000","last_clean":"2024-09-13T14:43:15.398165+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:43:15.398165+0000","last_undegraded":"2024-09-13T14:43:15.398165+0000","last_fullsized":"2024-09-13T14:43:15.398165+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317194240,"num_objects":4840,"num_object_clones":0,"num_object_copies":14520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6824,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":472,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673592+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:43:06.673592+0000","last_peered":"2024-09-13T14:43:06.673592+0000","last_clean":"2024-09-13T14:43:06.673592+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:43:06.673592+0000","last_undegraded":"2024-09-13T14:43:06.673592+0000","last_fullsized":"2024-09-13T14:43:06.673592+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":535,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.664252+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:43:06.664252+0000","last_peered":"2024-09-13T14:43:06.664252+0000","last_clean":"2024-09-13T14:43:06.664252+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:43:06.664252+0000","last_undegraded":"2024-09-13T14:43:06.664252+0000","last_fullsized":"2024-09-13T14:43:06.664252+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":504,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657911+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:43:06.657911+0000","last_peered":"2024-09-13T14:43:06.657911+0000","last_clean":"2024-09-13T14:43:06.657911+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:43:06.657911+0000","last_undegraded":"2024-09-13T14:43:06.657911+0000","last_fullsized":"2024-09-13T14:43:06.657911+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657022+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:43:06.657022+0000","last_peered":"2024-09-13T14:43:06.657022+0000","last_clean":"2024-09-13T14:43:06.657022+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:43:06.657022+0000","last_undegraded":"2024-09-13T14:43:06.657022+0000","last_fullsized":"2024-09-13T14:43:06.657022+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"340'6556","reported_seq":6641,"reported_epoch":340,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:15.548694+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:43:15.548694+0000","last_peered":"2024-09-13T14:43:15.548694+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:43:15.548694+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6556,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-835}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":149,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656845+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:43:06.656845+0000","last_peered":"2024-09-13T14:43:06.656845+0000","last_clean":"2024-09-13T14:43:06.656845+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:43:06.656845+0000","last_undegraded":"2024-09-13T14:43:06.656845+0000","last_fullsized":"2024-09-13T14:43:06.656845+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":956,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657181+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:43:06.657181+0000","last_peered":"2024-09-13T14:43:06.657181+0000","last_clean":"2024-09-13T14:43:06.657181+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:43:06.657181+0000","last_undegraded":"2024-09-13T14:43:06.657181+0000","last_fullsized":"2024-09-13T14:43:06.657181+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739884+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:43:06.739884+0000","last_peered":"2024-09-13T14:43:06.739884+0000","last_clean":"2024-09-13T14:43:06.739884+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:43:06.739884+0000","last_undegraded":"2024-09-13T14:43:06.739884+0000","last_fullsized":"2024-09-13T14:43:06.739884+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":525,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657876+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:43:06.657876+0000","last_peered":"2024-09-13T14:43:06.657876+0000","last_clean":"2024-09-13T14:43:06.657876+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:43:06.657876+0000","last_undegraded":"2024-09-13T14:43:06.657876+0000","last_fullsized":"2024-09-13T14:43:06.657876+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"341'6703","reported_seq":16071,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:18.497931+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:43:18.497931+0000","last_peered":"2024-09-13T14:43:18.497931+0000","last_clean":"2024-09-13T14:43:18.497931+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:43:18.497931+0000","last_undegraded":"2024-09-13T14:43:18.497931+0000","last_fullsized":"2024-09-13T14:43:18.497931+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305201152,"num_objects":4657,"num_object_clones":0,"num_object_copies":13971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6703,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818640+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:43:06.818640+0000","last_peered":"2024-09-13T14:43:06.818640+0000","last_clean":"2024-09-13T14:43:06.818640+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:43:06.818640+0000","last_undegraded":"2024-09-13T14:43:06.818640+0000","last_fullsized":"2024-09-13T14:43:06.818640+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":286,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657338+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:43:06.657338+0000","last_peered":"2024-09-13T14:43:06.657338+0000","last_clean":"2024-09-13T14:43:06.657338+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:43:06.657338+0000","last_undegraded":"2024-09-13T14:43:06.657338+0000","last_fullsized":"2024-09-13T14:43:06.657338+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1104,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658027+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:43:06.658027+0000","last_peered":"2024-09-13T14:43:06.658027+0000","last_clean":"2024-09-13T14:43:06.658027+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:43:06.658027+0000","last_undegraded":"2024-09-13T14:43:06.658027+0000","last_fullsized":"2024-09-13T14:43:06.658027+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":506,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658472+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:43:06.658472+0000","last_peered":"2024-09-13T14:43:06.658472+0000","last_clean":"2024-09-13T14:43:06.658472+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:43:06.658472+0000","last_undegraded":"2024-09-13T14:43:06.658472+0000","last_fullsized":"2024-09-13T14:43:06.658472+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"340'6618","reported_seq":6704,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:15.584186+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:43:15.584186+0000","last_peered":"2024-09-13T14:43:15.584186+0000","last_clean":"2024-09-13T14:43:15.584186+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:43:15.584186+0000","last_undegraded":"2024-09-13T14:43:15.584186+0000","last_fullsized":"2024-09-13T14:43:15.584186+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6618,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.817080+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:43:06.817080+0000","last_peered":"2024-09-13T14:43:06.817080+0000","last_clean":"2024-09-13T14:43:06.817080+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:43:06.817080+0000","last_undegraded":"2024-09-13T14:43:06.817080+0000","last_fullsized":"2024-09-13T14:43:06.817080+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":515,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657933+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:43:06.657933+0000","last_peered":"2024-09-13T14:43:06.657933+0000","last_clean":"2024-09-13T14:43:06.657933+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:43:06.657933+0000","last_undegraded":"2024-09-13T14:43:06.657933+0000","last_fullsized":"2024-09-13T14:43:06.657933+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":735,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674192+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:43:06.674192+0000","last_peered":"2024-09-13T14:43:06.674192+0000","last_clean":"2024-09-13T14:43:06.674192+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:43:06.674192+0000","last_undegraded":"2024-09-13T14:43:06.674192+0000","last_fullsized":"2024-09-13T14:43:06.674192+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":552,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733454+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:43:16.733454+0000","last_peered":"2024-09-13T14:43:16.733454+0000","last_clean":"2024-09-13T14:43:16.733454+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:43:16.733454+0000","last_undegraded":"2024-09-13T14:43:16.733454+0000","last_fullsized":"2024-09-13T14:43:16.733454+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"341'6945","reported_seq":7033,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:18.583417+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:43:18.583417+0000","last_peered":"2024-09-13T14:43:18.583417+0000","last_clean":"2024-09-13T14:43:18.583417+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:43:18.583417+0000","last_undegraded":"2024-09-13T14:43:18.583417+0000","last_fullsized":"2024-09-13T14:43:18.583417+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6945,"log_dups_size":0,"ondisk_log_size":6945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310968320,"num_objects":4745,"num_object_clones":0,"num_object_copies":14235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6945,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740022+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:43:06.740022+0000","last_peered":"2024-09-13T14:43:06.740022+0000","last_clean":"2024-09-13T14:43:06.740022+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:43:06.740022+0000","last_undegraded":"2024-09-13T14:43:06.740022+0000","last_fullsized":"2024-09-13T14:43:06.740022+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":550,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.665971+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:43:06.665971+0000","last_peered":"2024-09-13T14:43:06.665971+0000","last_clean":"2024-09-13T14:43:06.665971+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:43:06.665971+0000","last_undegraded":"2024-09-13T14:43:06.665971+0000","last_fullsized":"2024-09-13T14:43:06.665971+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":890,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658253+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:43:06.658253+0000","last_peered":"2024-09-13T14:43:06.658253+0000","last_clean":"2024-09-13T14:43:06.658253+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:43:06.658253+0000","last_undegraded":"2024-09-13T14:43:06.658253+0000","last_fullsized":"2024-09-13T14:43:06.658253+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":548,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.672094+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:43:06.672094+0000","last_peered":"2024-09-13T14:43:06.672094+0000","last_clean":"2024-09-13T14:43:06.672094+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:43:06.672094+0000","last_undegraded":"2024-09-13T14:43:06.672094+0000","last_fullsized":"2024-09-13T14:43:06.672094+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"341'6831","reported_seq":18183,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:17.343699+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:43:17.343699+0000","last_peered":"2024-09-13T14:43:17.343699+0000","last_clean":"2024-09-13T14:43:17.343699+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:43:17.343699+0000","last_undegraded":"2024-09-13T14:43:17.343699+0000","last_fullsized":"2024-09-13T14:43:17.343699+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306839576,"num_objects":4683,"num_object_clones":0,"num_object_copies":14049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4683,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6831,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":402,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.664834+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:43:06.664834+0000","last_peered":"2024-09-13T14:43:06.664834+0000","last_clean":"2024-09-13T14:43:06.664834+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:43:06.664834+0000","last_undegraded":"2024-09-13T14:43:06.664834+0000","last_fullsized":"2024-09-13T14:43:06.664834+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":588,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734212+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:43:16.734212+0000","last_peered":"2024-09-13T14:43:16.734212+0000","last_clean":"2024-09-13T14:43:16.734212+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:43:16.734212+0000","last_undegraded":"2024-09-13T14:43:16.734212+0000","last_fullsized":"2024-09-13T14:43:16.734212+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":759,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818724+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:43:06.818724+0000","last_peered":"2024-09-13T14:43:06.818724+0000","last_clean":"2024-09-13T14:43:06.818724+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:43:06.818724+0000","last_undegraded":"2024-09-13T14:43:06.818724+0000","last_fullsized":"2024-09-13T14:43:06.818724+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739650+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:43:06.739650+0000","last_peered":"2024-09-13T14:43:06.739650+0000","last_clean":"2024-09-13T14:43:06.739650+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:43:06.739650+0000","last_undegraded":"2024-09-13T14:43:06.739650+0000","last_fullsized":"2024-09-13T14:43:06.739650+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739936+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:43:06.739936+0000","last_peered":"2024-09-13T14:43:06.739936+0000","last_clean":"2024-09-13T14:43:06.739936+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:43:06.739936+0000","last_undegraded":"2024-09-13T14:43:06.739936+0000","last_fullsized":"2024-09-13T14:43:06.739936+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":50,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739892+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:43:06.739892+0000","last_peered":"2024-09-13T14:43:06.739892+0000","last_clean":"2024-09-13T14:43:06.739892+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:43:06.739892+0000","last_undegraded":"2024-09-13T14:43:06.739892+0000","last_fullsized":"2024-09-13T14:43:06.739892+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":502,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658324+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:43:06.658324+0000","last_peered":"2024-09-13T14:43:06.658324+0000","last_clean":"2024-09-13T14:43:06.658324+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:43:06.658324+0000","last_undegraded":"2024-09-13T14:43:06.658324+0000","last_fullsized":"2024-09-13T14:43:06.658324+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674107+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:43:06.674107+0000","last_peered":"2024-09-13T14:43:06.674107+0000","last_clean":"2024-09-13T14:43:06.674107+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:43:06.674107+0000","last_undegraded":"2024-09-13T14:43:06.674107+0000","last_fullsized":"2024-09-13T14:43:06.674107+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656999+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:43:06.656999+0000","last_peered":"2024-09-13T14:43:06.656999+0000","last_clean":"2024-09-13T14:43:06.656999+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:43:06.656999+0000","last_undegraded":"2024-09-13T14:43:06.656999+0000","last_fullsized":"2024-09-13T14:43:06.656999+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":507,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.663994+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:43:06.663994+0000","last_peered":"2024-09-13T14:43:06.663994+0000","last_clean":"2024-09-13T14:43:06.663994+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:43:06.663994+0000","last_undegraded":"2024-09-13T14:43:06.663994+0000","last_fullsized":"2024-09-13T14:43:06.663994+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":560,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733501+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:43:16.733501+0000","last_peered":"2024-09-13T14:43:16.733501+0000","last_clean":"2024-09-13T14:43:16.733501+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:43:16.733501+0000","last_undegraded":"2024-09-13T14:43:16.733501+0000","last_fullsized":"2024-09-13T14:43:16.733501+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":563,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.672030+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:43:06.672030+0000","last_peered":"2024-09-13T14:43:06.672030+0000","last_clean":"2024-09-13T14:43:06.672030+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:43:06.672030+0000","last_undegraded":"2024-09-13T14:43:06.672030+0000","last_fullsized":"2024-09-13T14:43:06.672030+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":338,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818681+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:43:06.818681+0000","last_peered":"2024-09-13T14:43:06.818681+0000","last_clean":"2024-09-13T14:43:06.818681+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:43:06.818681+0000","last_undegraded":"2024-09-13T14:43:06.818681+0000","last_fullsized":"2024-09-13T14:43:06.818681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":580,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.672115+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:43:06.672115+0000","last_peered":"2024-09-13T14:43:06.672115+0000","last_clean":"2024-09-13T14:43:06.672115+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:43:06.672115+0000","last_undegraded":"2024-09-13T14:43:06.672115+0000","last_fullsized":"2024-09-13T14:43:06.672115+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":552,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734266+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:43:16.734266+0000","last_peered":"2024-09-13T14:43:16.734266+0000","last_clean":"2024-09-13T14:43:16.734266+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:43:16.734266+0000","last_undegraded":"2024-09-13T14:43:16.734266+0000","last_fullsized":"2024-09-13T14:43:16.734266+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":527,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657995+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:43:06.657995+0000","last_peered":"2024-09-13T14:43:06.657995+0000","last_clean":"2024-09-13T14:43:06.657995+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:43:06.657995+0000","last_undegraded":"2024-09-13T14:43:06.657995+0000","last_fullsized":"2024-09-13T14:43:06.657995+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":576,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.665145+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:43:06.665145+0000","last_peered":"2024-09-13T14:43:06.665145+0000","last_clean":"2024-09-13T14:43:06.665145+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:43:06.665145+0000","last_undegraded":"2024-09-13T14:43:06.665145+0000","last_fullsized":"2024-09-13T14:43:06.665145+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":39,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.747372+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:43:06.747372+0000","last_peered":"2024-09-13T14:43:06.747372+0000","last_clean":"2024-09-13T14:43:06.747372+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:43:06.747372+0000","last_undegraded":"2024-09-13T14:43:06.747372+0000","last_fullsized":"2024-09-13T14:43:06.747372+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":473,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.658283+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:43:06.658283+0000","last_peered":"2024-09-13T14:43:06.658283+0000","last_clean":"2024-09-13T14:43:06.658283+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:43:06.658283+0000","last_undegraded":"2024-09-13T14:43:06.658283+0000","last_fullsized":"2024-09-13T14:43:06.658283+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":556,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733832+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:43:16.733832+0000","last_peered":"2024-09-13T14:43:16.733832+0000","last_clean":"2024-09-13T14:43:16.733832+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:43:16.733832+0000","last_undegraded":"2024-09-13T14:43:16.733832+0000","last_fullsized":"2024-09-13T14:43:16.733832+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739861+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:43:06.739861+0000","last_peered":"2024-09-13T14:43:06.739861+0000","last_clean":"2024-09-13T14:43:06.739861+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:43:06.739861+0000","last_undegraded":"2024-09-13T14:43:06.739861+0000","last_fullsized":"2024-09-13T14:43:06.739861+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657271+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:43:06.657271+0000","last_peered":"2024-09-13T14:43:06.657271+0000","last_clean":"2024-09-13T14:43:06.657271+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:43:06.657271+0000","last_undegraded":"2024-09-13T14:43:06.657271+0000","last_fullsized":"2024-09-13T14:43:06.657271+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":39,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.747284+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:43:06.747284+0000","last_peered":"2024-09-13T14:43:06.747284+0000","last_clean":"2024-09-13T14:43:06.747284+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:43:06.747284+0000","last_undegraded":"2024-09-13T14:43:06.747284+0000","last_fullsized":"2024-09-13T14:43:06.747284+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":272,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657430+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:43:06.657430+0000","last_peered":"2024-09-13T14:43:06.657430+0000","last_clean":"2024-09-13T14:43:06.657430+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:43:06.657430+0000","last_undegraded":"2024-09-13T14:43:06.657430+0000","last_fullsized":"2024-09-13T14:43:06.657430+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":572,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.664347+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:43:06.664347+0000","last_peered":"2024-09-13T14:43:06.664347+0000","last_clean":"2024-09-13T14:43:06.664347+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:43:06.664347+0000","last_undegraded":"2024-09-13T14:43:06.664347+0000","last_fullsized":"2024-09-13T14:43:06.664347+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":559,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.665509+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:43:06.665509+0000","last_peered":"2024-09-13T14:43:06.665509+0000","last_clean":"2024-09-13T14:43:06.665509+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:43:06.665509+0000","last_undegraded":"2024-09-13T14:43:06.665509+0000","last_fullsized":"2024-09-13T14:43:06.665509+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739544+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:43:06.739544+0000","last_peered":"2024-09-13T14:43:06.739544+0000","last_clean":"2024-09-13T14:43:06.739544+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:43:06.739544+0000","last_undegraded":"2024-09-13T14:43:06.739544+0000","last_fullsized":"2024-09-13T14:43:06.739544+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":566,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733616+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:43:16.733616+0000","last_peered":"2024-09-13T14:43:16.733616+0000","last_clean":"2024-09-13T14:43:16.733616+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:43:16.733616+0000","last_undegraded":"2024-09-13T14:43:16.733616+0000","last_fullsized":"2024-09-13T14:43:16.733616+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4925620248,"num_objects":75160,"num_object_clones":0,"num_object_copies":225480,"num_objects_missing_on_primary":3665,"num_objects_missing":3665,"num_objects_degraded":40025,"num_objects_misplaced":4617,"num_objects_unfound":0,"num_objects_dirty":75160,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107518,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21785,"num_bytes_recovered":1367277568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13482995712,"data_stored":13482983496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107518,"ondisk_log_size":107518,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397886,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2656592,"kb_used_data":1764740,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91088560,"statfs":{"total":95995035648,"available":93274685440,"internally_reserved":0,"allocated":1807093760,"data_stored":1805324974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594114,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2451452,"kb_used_data":1861552,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":91293700,"statfs":{"total":95995035648,"available":93484748800,"internally_reserved":0,"allocated":1906229248,"data_stored":1904480216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665568,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":675452,"kb_used_data":351216,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93069700,"statfs":{"total":95995035648,"available":95303372800,"internally_reserved":0,"allocated":359645184,"data_stored":357892483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986570,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1945608,"kb_used_data":1045756,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":91799544,"statfs":{"total":95995035648,"available":94002733056,"internally_reserved":0,"allocated":1070854144,"data_stored":1069093456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,5,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697129,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2902308,"kb_used_data":2180252,"kb_used_omap":4,"kb_used_meta":722043,"kb_avail":90842844,"statfs":{"total":95995035648,"available":93023072256,"internally_reserved":0,"allocated":2232578048,"data_stored":2230837347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":739372107},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860656,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2722176,"kb_used_data":2096756,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91022976,"statfs":{"total":95995035648,"available":93207527424,"internally_reserved":0,"allocated":2147078144,"data_stored":2145321942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020327,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2422944,"kb_used_data":2115224,"kb_used_omap":9,"kb_used_meta":307702,"kb_avail":91322208,"statfs":{"total":95995035648,"available":93513940992,"internally_reserved":0,"allocated":2165989376,"data_stored":2164252148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":315087685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187718,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2418992,"kb_used_data":1808488,"kb_used_omap":7,"kb_used_meta":610488,"kb_avail":91326160,"statfs":{"total":95995035648,"available":93517987840,"internally_reserved":0,"allocated":1851891712,"data_stored":1850146903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":625140043},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1843265536,"data_stored":1843265536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2156199936,"data_stored":2156199936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2142048256,"data_stored":2142044184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2223833088,"data_stored":2223833088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1065287680,"data_stored":1065287680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":354750464,"data_stored":354746392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1900871680,"data_stored":1900871680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1796739072,"data_stored":1796735000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:21.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:21.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:21.166 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:21.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.240 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:21.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:21.466 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:21.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:21.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:43:21.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:21.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:43:21.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:21.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:21.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:21.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:21.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:21.895 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:21 smithi017 ceph-mon[103001]: from='client.46064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:21 smithi017 ceph-mon[107556]: from='client.46064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:21.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:21.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:21 smithi189 ceph-mon[90992]: from='client.46064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:22.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:22.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:22.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:22.241 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:22.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:22.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:22.364 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:22.408 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:22.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:22.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:22.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:22.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:22.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:22.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:22.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:22.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:22.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:22.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:22 smithi017 ceph-mon[107556]: pgmap v1017: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 40025/224613 objects degraded (17.820%); 4617/224613 objects misplaced (2.056%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:22.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:22 smithi017 ceph-mon[103001]: pgmap v1017: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 40025/224613 objects degraded (17.820%); 4617/224613 objects misplaced (2.056%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:22.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:22.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:22.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:22 smithi189 ceph-mon[90992]: pgmap v1017: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 40025/224613 objects degraded (17.820%); 4617/224613 objects misplaced (2.056%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:43:23.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:23.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:23.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:23.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:23.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:23.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:23.466 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.490 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:23.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:23.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.743 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:23 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 40025/224613 objects degraded (17.820%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:23.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:23 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 40025/224613 objects degraded (17.820%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:23.772 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:23 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 40025/224613 objects degraded (17.820%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:23.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:23.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:23.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:23.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:24.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:24.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:24.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:24.051 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:24.051 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:24.054 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:24.055 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:24.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:43:24.072 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:24.072 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:24.088 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:24.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:24.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:24.104 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:24.104 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:24.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:24.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:24.120 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:24.120 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:24.151 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:24.151 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:24.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:24.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:24.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:24.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:24.490 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:24.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:24.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:24.709 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:43:24.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:24.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:24 smithi017 ceph-mon[103001]: pgmap v1018: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39961/224208 objects degraded (17.823%); 4617/224208 objects misplaced (2.059%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:24 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/202855072' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:24 smithi017 ceph-mon[107556]: pgmap v1018: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39961/224208 objects degraded (17.823%); 4617/224208 objects misplaced (2.059%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:24 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/202855072' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:24.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:24.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:25.001 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:24 smithi189 ceph-mon[90992]: pgmap v1018: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39961/224208 objects degraded (17.823%); 4617/224208 objects misplaced (2.059%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:24 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/202855072' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:25.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:25.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:25.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.293 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:25.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:25.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:25.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:25.700 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:43:25.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:25.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:25.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:25.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:25.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:25.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:25.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:25.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:26.098 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:26.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:26.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:26.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:26.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:26.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:26.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:26.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:26.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:26.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:26.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:26.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:26 smithi189 ceph-mon[90992]: pgmap v1019: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39925/223467 objects degraded (17.866%); 4583/223467 objects misplaced (2.051%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:26 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:26 smithi189 ceph-mon[90992]: osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:43:26.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:26.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[103001]: pgmap v1019: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39925/223467 objects degraded (17.866%); 4583/223467 objects misplaced (2.051%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[103001]: osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[107556]: pgmap v1019: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39925/223467 objects degraded (17.866%); 4583/223467 objects misplaced (2.051%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3654917332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:26 smithi017 ceph-mon[107556]: osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:43:26.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:26.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:27.096 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:43:27.096 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:43:27.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:43:27.096 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:43:27.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:198597 6.12 6.510888d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e206)", 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:34.350463+0000", 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 592.71061066200002, 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89083111199999998, 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 198597 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.097 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350463+0000", 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350463+0000", 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350461+0000", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350470+0000", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2879999999999993e-06 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350484+0000", 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4588e-05 2024-09-13T14:43:27.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350491+0000", 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6510000000000004e-06 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350531+0000", 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9762000000000003e-05 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:34.350538+0000", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.878e-06 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:35.241284+0000", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89074635099999999 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:35.241294+0000", 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0071999999999999e-05 2024-09-13T14:43:27.101 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202401 6.12 6:4d34b7de:::benchmark_data_smithi017_185083_object77789:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.340239+0000", 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.72083459999999, 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.079851866, 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202401 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340239+0000", 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340239+0000", 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340238+0000", 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340243+0000", 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3979999999999998e-06 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340255+0000", 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1671000000000001e-05 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340259+0000", 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5519999999999998e-06 2024-09-13T14:43:27.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340293+0000", 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3745999999999998e-05 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340305+0000", 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1537e-05 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.410960+0000", 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.070655183 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.411056+0000", 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6111000000000002e-05 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420026+0000", 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0089703209999999999 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420091+0000", 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4374000000000004e-05 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202410 6.12 6:486d3b1b:::benchmark_data_smithi017_185083_object77798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.340310+0000", 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.72076384100001, 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.079600812, 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202410 2024-09-13T14:43:27.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340310+0000", 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340310+0000", 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340309+0000", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340312+0000", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.976e-06 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340321+0000", 2024-09-13T14:43:27.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.568e-06 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340325+0000", 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.549e-06 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340348+0000", 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2303e-05 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.340352+0000", 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8579999999999999e-06 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.416049+0000", 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.075696255 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.416136+0000", 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7163999999999994e-05 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.416390+0000", 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025456300000000002 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.419703+0000", 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003312845 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.419709+0000", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6359999999999999e-06 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.419800+0000", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0781999999999999e-05 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.419834+0000", 2024-09-13T14:43:27.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4654999999999999e-05 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.419910+0000", 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6174000000000006e-05 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202445 6.12 6:48170c41:::benchmark_data_smithi017_185083_object77833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.371307+0000", 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.68976600899998, 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0532536429999999, 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202445 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.117 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371307+0000", 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371307+0000", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371306+0000", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371313+0000", 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7880000000000001e-06 2024-09-13T14:43:27.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371326+0000", 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3154e-05 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371332+0000", 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8769999999999998e-06 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371420+0000", 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8509000000000002e-05 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371429+0000", 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9500000000000007e-06 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.418577+0000", 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.047147515 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.418704+0000", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000127656 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.418924+0000", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000219426 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420090+0000", 2024-09-13T14:43:27.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001166121 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.421353+0000", 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001262998 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424425+0000", 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030721530000000002 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424470+0000", 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4861000000000003e-05 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424561+0000", 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1224999999999995e-05 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.124 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202446 6.12 6:4a00422b:::benchmark_data_smithi017_185083_object77834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.371504+0000", 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.68956914099999, 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0568886040000001, 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202446 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.125 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371504+0000", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371504+0000", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371503+0000", 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.126 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371509+0000", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.48e-06 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371524+0000", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5116000000000001e-05 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371530+0000", 2024-09-13T14:43:27.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6110000000000001e-06 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371567+0000", 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7802000000000002e-05 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.371576+0000", 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.636e-06 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420272+0000", 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0486962369999999 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420334+0000", 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1456000000000002e-05 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.129 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.420465+0000", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013159399999999999 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424613+0000", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041477759999999997 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424647+0000", 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4110999999999999e-05 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428245+0000", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003597561 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428296+0000", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1205999999999997e-05 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428393+0000", 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6947000000000005e-05 2024-09-13T14:43:27.131 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202466 6.12 6:49feaf98:::benchmark_data_smithi017_185083_object77854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.388869+0000", 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.67220420700005, 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0397165850000001, 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.132 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202466 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388869+0000", 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388869+0000", 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388868+0000", 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388875+0000", 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7710000000000001e-06 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388887+0000", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.204e-05 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388892+0000", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2170000000000002e-06 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388935+0000", 2024-09-13T14:43:27.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2797999999999999e-05 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.388943+0000", 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2989999999999997e-06 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.421435+0000", 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.032491898 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.421671+0000", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00023648000000000001 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.421915+0000", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000243856 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424680+0000", 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027644470000000002 2024-09-13T14:43:27.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428403+0000", 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037235770000000001 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428452+0000", 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8897999999999998e-05 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428498+0000", 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6004999999999997e-05 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428586+0000", 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7715999999999998e-05 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202503 6.12 6:4ede620b:::benchmark_data_smithi017_185083_object77891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.139 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.420178+0000", 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.64089574800005, 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0086510500000001, 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202503 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.140 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420178+0000", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420178+0000", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420177+0000", 2024-09-13T14:43:27.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420181+0000", 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6199999999999998e-06 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420191+0000", 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5079999999999994e-06 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420196+0000", 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3199999999999999e-06 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420237+0000", 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0481000000000001e-05 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.420243+0000", 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9539999999999998e-06 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.422900+0000", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.00265647 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.422968+0000", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8167e-05 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.423151+0000", 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000183186 2024-09-13T14:43:27.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428627+0000", 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0054755150000000002 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428634+0000", 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3819999999999998e-06 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428677+0000", 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3309999999999997e-05 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428732+0000", 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4629000000000002e-05 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428829+0000", 2024-09-13T14:43:27.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6667000000000001e-05 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202516 6.12 6:4a6bd69a:::benchmark_data_smithi017_185083_object77904:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.431894+0000", 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.62917914599996, 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99718411100000004, 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.147 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202516 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431894+0000", 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.148 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431894+0000", 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431893+0000", 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431899+0000", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.818e-06 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431912+0000", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3441000000000001e-05 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431918+0000", 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.964e-06 2024-09-13T14:43:27.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431967+0000", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8649999999999997e-05 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.431976+0000", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1300000000000007e-06 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424719+0000", 2024-09-13T14:43:27.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99274263299999999 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.424838+0000", 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000119098 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.425050+0000", 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021206 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428638+0000", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003588146 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428897+0000", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025899899999999999 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428954+0000", 2024-09-13T14:43:27.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7089999999999997e-05 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428996+0000", 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1878999999999997e-05 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.429078+0000", 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2392000000000007e-05 2024-09-13T14:43:27.154 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:202544 6.12 6:4e12df40:::benchmark_data_smithi017_185083_object77932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e212)", 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:41.459888+0000", 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 585.60118534000003, 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97814213500000002, 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 202544 2024-09-13T14:43:27.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459888+0000", 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459888+0000", 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459887+0000", 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.157 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459893+0000", 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7359999999999998e-06 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459905+0000", 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1952e-05 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459911+0000", 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9070000000000004e-06 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459956+0000", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5148000000000002e-05 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:41.459964+0000", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7050000000000005e-06 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.426508+0000", 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96654386199999998 2024-09-13T14:43:27.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.426609+0000", 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101065 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.426862+0000", 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025215799999999999 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.428910+0000", 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0020480540000000001 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.429130+0000", 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000220557 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.437900+0000", 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0087697899999999995 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.437948+0000", 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8068000000000001e-05 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:42.438030+0000", 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2249000000000005e-05 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:210513 6.15 6:ac79758a:::benchmark_data_smithi017_185083_object85901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:54.630263+0000", 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 572.43081056599999, 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96846995300000005, 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 210513 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630263+0000", 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630263+0000", 2024-09-13T14:43:27.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.165 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.165 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.165 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.165 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630261+0000", 2024-09-13T14:43:27.165 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630270+0000", 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6049999999999998e-06 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630292+0000", 2024-09-13T14:43:27.166 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2141999999999999e-05 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630301+0000", 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7770000000000008e-06 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630356+0000", 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5278999999999997e-05 2024-09-13T14:43:27.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.630366+0000", 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0502999999999999e-05 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.579958+0000", 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94959171600000003 2024-09-13T14:43:27.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.580066+0000", 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000107668 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.580300+0000", 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00023438800000000001 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.586073+0000", 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057732510000000001 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598086+0000", 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012012907999999999 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598660+0000", 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00057321399999999995 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598686+0000", 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6665e-05 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598733+0000", 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6621e-05 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.171 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:210516 6.15 6:ac16a81a:::benchmark_data_smithi017_185083_object85904:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:54.635262+0000", 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 572.425811297, 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96359914300000005, 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.172 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 210516 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635262+0000", 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.173 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635262+0000", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635260+0000", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635272+0000", 2024-09-13T14:43:27.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2288e-05 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635293+0000", 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0832999999999999e-05 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.175 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635302+0000", 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2090000000000005e-06 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635361+0000", 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9095999999999998e-05 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.176 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.635370+0000", 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9320000000000002e-06 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.581826+0000", 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94645614600000005 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.581912+0000", 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5804999999999996e-05 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.582075+0000", 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000163098 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.588718+0000", 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066433250000000003 2024-09-13T14:43:27.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598764+0000", 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010045722999999999 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598779+0000", 2024-09-13T14:43:27.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5761999999999999e-05 2024-09-13T14:43:27.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598803+0000", 2024-09-13T14:43:27.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3158000000000001e-05 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598861+0000", 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8761999999999997e-05 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35349.0:210535 6.15 6:aa74e30a:::benchmark_data_smithi017_185083_object85923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:33:54.656543+0000", 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 572.404530142, 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.942510862, 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35349", 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/486706043", 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 210535 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.183 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656543+0000", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656543+0000", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656542+0000", 2024-09-13T14:43:27.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656548+0000", 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.835e-06 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656560+0000", 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.22e-05 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656568+0000", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1149999999999996e-06 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656623+0000", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5757000000000002e-05 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:54.656631+0000", 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2590000000000003e-06 2024-09-13T14:43:27.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.583045+0000", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92641430400000002 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.583110+0000", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5554000000000003e-05 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.583250+0000", 2024-09-13T14:43:27.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000139801 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.588725+0000", 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0054746020000000003 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598880+0000", 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010155047 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598923+0000", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3355000000000001e-05 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.598971+0000", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7760999999999999e-05 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:33:55.599054+0000", 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3196999999999996e-05 2024-09-13T14:43:27.189 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:221968 7.11 7:8fda2ab3:::benchmark_data_smithi017_220393_object102834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.161858+0000", 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.89921592500002, 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.011231633, 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 221968 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161858+0000", 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161858+0000", 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161856+0000", 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161864+0000", 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7639999999999994e-06 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161879+0000", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4448e-05 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161886+0000", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9419999999999998e-06 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161929+0000", 2024-09-13T14:43:27.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2920000000000002e-05 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.161936+0000", 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8839999999999997e-06 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.169650+0000", 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0077145750000001 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.169740+0000", 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0235999999999995e-05 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.169813+0000", 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2300999999999998e-05 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.172841+0000", 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030280900000000002 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.172852+0000", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0927999999999999e-05 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173005+0000", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015310599999999999 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173051+0000", 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5738999999999997e-05 2024-09-13T14:43:27.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173089+0000", 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8532000000000003e-05 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.197 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222002 7.11 7:8d382ff1:::benchmark_data_smithi017_220393_object102868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.189571+0000", 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.87150203099998, 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98360877800000002, 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.198 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222002 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189571+0000", 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189571+0000", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189570+0000", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189577+0000", 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8090000000000004e-06 2024-09-13T14:43:27.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189589+0000", 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1782e-05 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.201 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189593+0000", 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4789999999999998e-06 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189637+0000", 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4302999999999999e-05 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.202 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.203 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.189646+0000", 2024-09-13T14:43:27.203 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1079999999999999e-06 2024-09-13T14:43:27.205 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.205 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.205 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.205 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.171083+0000", 2024-09-13T14:43:27.206 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.981436321 2024-09-13T14:43:27.206 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.206 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.171141+0000", 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8143999999999998e-05 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.171199+0000", 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7938e-05 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.207 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.208 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.172880+0000", 2024-09-13T14:43:27.208 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001681613 2024-09-13T14:43:27.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173128+0000", 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024788700000000002 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173133+0000", 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7559999999999999e-06 2024-09-13T14:43:27.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173159+0000", 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6129999999999999e-05 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173180+0000", 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1053e-05 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.210 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222009 7.11 7:8ab0d4b5:::benchmark_data_smithi017_220393_object102875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.192921+0000", 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.86815251299998, 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98038813700000005, 2024-09-13T14:43:27.211 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222009 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192921+0000", 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192921+0000", 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192920+0000", 2024-09-13T14:43:27.213 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192925+0000", 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9300000000000002e-06 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.214 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192937+0000", 2024-09-13T14:43:27.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:27.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:27.217 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1620999999999999e-05 2024-09-13T14:43:27.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192941+0000", 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4059999999999998e-06 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192984+0000", 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2735000000000003e-05 2024-09-13T14:43:27.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.192998+0000", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4518999999999999e-05 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.171912+0000", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97891341700000001 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.171978+0000", 2024-09-13T14:43:27.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6501999999999994e-05 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.172044+0000", 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5936e-05 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173186+0000", 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011415310000000001 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.220 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173213+0000", 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7858e-05 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173252+0000", 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8791999999999997e-05 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173292+0000", 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9779e-05 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.221 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173309+0000", 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7099999999999999e-05 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222017 7.11 7:8fe02de3:::benchmark_data_smithi017_220393_object102883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.199676+0000", 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.861397877, 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.983297325, 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.222 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222017 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199676+0000", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199676+0000", 2024-09-13T14:43:27.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199675+0000", 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199680+0000", 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6069999999999998e-06 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199694+0000", 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3328e-05 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199700+0000", 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3759999999999996e-06 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199761+0000", 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0492999999999998e-05 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.199769+0000", 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0120000000000005e-06 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173338+0000", 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97356980800000004 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173397+0000", 2024-09-13T14:43:27.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8301000000000003e-05 2024-09-13T14:43:27.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.227 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.227 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.173450+0000", 2024-09-13T14:43:27.228 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3285999999999998e-05 2024-09-13T14:43:27.228 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176661+0000", 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003211214 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176666+0000", 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4800000000000003e-06 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.182905+0000", 2024-09-13T14:43:27.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062390120000000004 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.182943+0000", 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8257999999999999e-05 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.182973+0000", 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0035e-05 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.230 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222026 7.11 7:8da0526b:::benchmark_data_smithi017_220393_object102892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.205618+0000", 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.85545589999998, 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97747928100000003, 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222026 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.231 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205618+0000", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205618+0000", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205616+0000", 2024-09-13T14:43:27.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205625+0000", 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2670000000000006e-06 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205638+0000", 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2818e-05 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205644+0000", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6719999999999998e-06 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205709+0000", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5131999999999999e-05 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.205716+0000", 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8480000000000003e-06 2024-09-13T14:43:27.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.174239+0000", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96852274400000005 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.174337+0000", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8148999999999998e-05 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.174427+0000", 2024-09-13T14:43:27.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9441e-05 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176708+0000", 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002281267 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.236 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.177620+0000", 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00091203300000000003 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183048+0000", 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005428581 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.237 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183074+0000", 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5318000000000001e-05 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183097+0000", 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3139e-05 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222042 7.11 7:8851860b:::benchmark_data_smithi017_220393_object102908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.218318+0000", 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.84275551100001, 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96488510999999999, 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222042 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.239 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218318+0000", 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218318+0000", 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218317+0000", 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218322+0000", 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7079999999999996e-06 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218334+0000", 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2092999999999999e-05 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218340+0000", 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8100000000000003e-06 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218403+0000", 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3619000000000002e-05 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.218411+0000", 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0860000000000002e-06 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.175429+0000", 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95701774500000003 2024-09-13T14:43:27.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.175520+0000", 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1273000000000005e-05 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.175586+0000", 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5858000000000005e-05 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.244 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.178400+0000", 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028134869999999999 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.178406+0000", 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4609999999999996e-06 2024-09-13T14:43:27.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183139+0000", 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047327189999999998 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183175+0000", 2024-09-13T14:43:27.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5889000000000001e-05 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.183203+0000", 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8328999999999999e-05 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222044 7.11 7:8ce9e208:::benchmark_data_smithi017_220393_object102910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.247 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.220066+0000", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.84100720800001, 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96444270300000001, 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222044 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220066+0000", 2024-09-13T14:43:27.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220066+0000", 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220066+0000", 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220069+0000", 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1999999999999999e-06 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220077+0000", 2024-09-13T14:43:27.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9529999999999999e-06 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220082+0000", 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8600000000000001e-06 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220117+0000", 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5256000000000003e-05 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.251 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.220123+0000", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0719999999999996e-06 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176730+0000", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95660694599999996 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176809+0000", 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9202999999999994e-05 2024-09-13T14:43:27.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.176900+0000", 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1119000000000006e-05 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.178434+0000", 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0015337370000000001 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.179528+0000", 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001094065 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184441+0000", 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0049124590000000001 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184478+0000", 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7398999999999999e-05 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184509+0000", 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1016999999999998e-05 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:222133 7.11 7:89533829:::benchmark_data_smithi017_220393_object102999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:58.291215+0000", 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 268.76985856599998, 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89338498899999996, 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:43:27.255 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 222133 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291215+0000", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:43:27.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291215+0000", 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291214+0000", 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291218+0000", 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1819999999999999e-06 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291228+0000", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0039e-05 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291233+0000", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0999999999999997e-06 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291272+0000", 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9104999999999999e-05 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:58.291279+0000", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3220000000000004e-06 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.177637+0000", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88635841400000004 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.177699+0000", 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1952000000000006e-05 2024-09-13T14:43:27.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,6", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.177767+0000", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7974999999999999e-05 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.179557+0000", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001789992 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.179562+0000", 2024-09-13T14:43:27.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8629999999999999e-06 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184538+0000", 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0049757400000000002 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184568+0000", 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0318999999999999e-05 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:43:27.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:59.184600+0000", 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1717000000000001e-05 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:43:27.262 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:43:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:27.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:27.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:27.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:27.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:27.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:27.497 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:27.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:27.586 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:27] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:43:27.621 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:27.643 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:27.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:27.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:27.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:27.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:27.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:27.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:27.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:27.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:27.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:27.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:28.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:43:28.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:28.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:28.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:28.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:28.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:28.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:28.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:28.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:28.401 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:28.402 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:43:28.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:28.521 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:28.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:28.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:28.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:28.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:28.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:43:28.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.895 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:28 smithi017 ceph-mon[107556]: pgmap v1021: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 39847/222945 objects degraded (17.873%); 4583/222945 objects misplaced (2.056%); 889 KiB/s, 13 objects/s recovering 2024-09-13T14:43:28.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:28 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39847/222945 objects degraded (17.873%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:28.981 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:28 smithi017 ceph-mon[103001]: pgmap v1021: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 39847/222945 objects degraded (17.873%); 4583/222945 objects misplaced (2.056%); 889 KiB/s, 13 objects/s recovering 2024-09-13T14:43:28.981 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:28 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39847/222945 objects degraded (17.873%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:28.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:28 smithi189 ceph-mon[90992]: pgmap v1021: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 39847/222945 objects degraded (17.873%); 4583/222945 objects misplaced (2.056%); 889 KiB/s, 13 objects/s recovering 2024-09-13T14:43:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:28 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39847/222945 objects degraded (17.873%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:29.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:29.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:29.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:29.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:29.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:29.317 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1021,"stamp":"2024-09-13T14:43:27.189992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4873095862,"num_objects":74315,"num_object_clones":0,"num_object_copies":222945,"num_objects_missing_on_primary":3629,"num_objects_missing":3629,"num_objects_degraded":39847,"num_objects_misplaced":4583,"num_objects_unfound":0,"num_objects_dirty":74315,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":109338,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21947,"num_bytes_recovered":1374324893,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110523,"ondisk_log_size":110523,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18064864,"kb_used_data":13080864,"kb_used_omap":44,"kb_used_meta":4983891,"kb_avail":731896352,"statfs":{"total":767960285184,"available":749461864448,"internally_reserved":0,"allocated":13394804736,"data_stored":13380771048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5103504631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73203712,"num_objects":-1117,"num_object_clones":0,"num_object_copies":-3351,"num_objects_missing_on_primary":-70,"num_objects_missing":-70,"num_objects_degraded":-212,"num_objects_misplaced":-68,"num_objects_unfound":0,"num_objects_dirty":-1117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1117,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002886"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":565,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902269+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:43:16.902269+0000","last_peered":"2024-09-13T14:43:16.902269+0000","last_clean":"2024-09-13T14:43:16.902269+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:43:16.902269+0000","last_undegraded":"2024-09-13T14:43:16.902269+0000","last_fullsized":"2024-09-13T14:43:16.902269+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":339,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673833+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:43:06.673833+0000","last_peered":"2024-09-13T14:43:06.673833+0000","last_clean":"2024-09-13T14:43:06.673833+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:43:06.673833+0000","last_undegraded":"2024-09-13T14:43:06.673833+0000","last_fullsized":"2024-09-13T14:43:06.673833+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657404+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:43:06.657404+0000","last_peered":"2024-09-13T14:43:06.657404+0000","last_clean":"2024-09-13T14:43:06.657404+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:43:06.657404+0000","last_undegraded":"2024-09-13T14:43:06.657404+0000","last_fullsized":"2024-09-13T14:43:06.657404+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":550,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733936+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:43:16.733936+0000","last_peered":"2024-09-13T14:43:16.733936+0000","last_clean":"2024-09-13T14:43:16.733936+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:43:16.733936+0000","last_undegraded":"2024-09-13T14:43:16.733936+0000","last_fullsized":"2024-09-13T14:43:16.733936+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":551,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734297+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:43:16.734297+0000","last_peered":"2024-09-13T14:43:16.734297+0000","last_clean":"2024-09-13T14:43:16.734297+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:43:16.734297+0000","last_undegraded":"2024-09-13T14:43:16.734297+0000","last_fullsized":"2024-09-13T14:43:16.734297+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":308,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673669+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:43:06.673669+0000","last_peered":"2024-09-13T14:43:06.673669+0000","last_clean":"2024-09-13T14:43:06.673669+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:43:06.673669+0000","last_undegraded":"2024-09-13T14:43:06.673669+0000","last_fullsized":"2024-09-13T14:43:06.673669+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":565,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902579+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:43:16.902579+0000","last_peered":"2024-09-13T14:43:16.902579+0000","last_clean":"2024-09-13T14:43:16.902579+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:43:16.902579+0000","last_undegraded":"2024-09-13T14:43:16.902579+0000","last_fullsized":"2024-09-13T14:43:16.902579+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":330,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816650+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:43:06.816650+0000","last_peered":"2024-09-13T14:43:06.816650+0000","last_clean":"2024-09-13T14:43:06.816650+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:43:06.816650+0000","last_undegraded":"2024-09-13T14:43:06.816650+0000","last_fullsized":"2024-09-13T14:43:06.816650+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"342'6718","reported_seq":10021,"reported_epoch":342,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:26.778842+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:43:26.778842+0000","last_peered":"2024-09-13T14:43:26.778842+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:43:26.778842+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6718,"log_dups_size":0,"ondisk_log_size":6718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299630592,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8059,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6718,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1073,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3588}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":334,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674328+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:43:06.674328+0000","last_peered":"2024-09-13T14:43:06.674328+0000","last_clean":"2024-09-13T14:43:06.674328+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:43:06.674328+0000","last_undegraded":"2024-09-13T14:43:06.674328+0000","last_fullsized":"2024-09-13T14:43:06.674328+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657054+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:43:06.657054+0000","last_peered":"2024-09-13T14:43:06.657054+0000","last_clean":"2024-09-13T14:43:06.657054+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:43:06.657054+0000","last_undegraded":"2024-09-13T14:43:06.657054+0000","last_fullsized":"2024-09-13T14:43:06.657054+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"341'6787","reported_seq":16325,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:22.847782+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:43:22.847782+0000","last_peered":"2024-09-13T14:43:22.847782+0000","last_clean":"2024-09-13T14:43:22.847782+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:43:22.847782+0000","last_undegraded":"2024-09-13T14:43:22.847782+0000","last_fullsized":"2024-09-13T14:43:22.847782+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6787,"log_dups_size":0,"ondisk_log_size":6787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6787,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674402+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:43:06.674402+0000","last_peered":"2024-09-13T14:43:06.674402+0000","last_clean":"2024-09-13T14:43:06.674402+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:43:06.674402+0000","last_undegraded":"2024-09-13T14:43:06.674402+0000","last_fullsized":"2024-09-13T14:43:06.674402+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739721+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:43:06.739721+0000","last_peered":"2024-09-13T14:43:06.739721+0000","last_clean":"2024-09-13T14:43:06.739721+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:43:06.739721+0000","last_undegraded":"2024-09-13T14:43:06.739721+0000","last_fullsized":"2024-09-13T14:43:06.739721+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"341'7000","reported_seq":7088,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:22.941161+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:43:22.941161+0000","last_peered":"2024-09-13T14:43:22.941161+0000","last_clean":"2024-09-13T14:43:22.941161+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:43:22.941161+0000","last_undegraded":"2024-09-13T14:43:22.941161+0000","last_fullsized":"2024-09-13T14:43:22.941161+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314441728,"num_objects":4798,"num_object_clones":0,"num_object_copies":14394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7000,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674296+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:43:06.674296+0000","last_peered":"2024-09-13T14:43:06.674296+0000","last_clean":"2024-09-13T14:43:06.674296+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:43:06.674296+0000","last_undegraded":"2024-09-13T14:43:06.674296+0000","last_fullsized":"2024-09-13T14:43:06.674296+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":321,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816842+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:43:06.816842+0000","last_peered":"2024-09-13T14:43:06.816842+0000","last_clean":"2024-09-13T14:43:06.816842+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:43:06.816842+0000","last_undegraded":"2024-09-13T14:43:06.816842+0000","last_fullsized":"2024-09-13T14:43:06.816842+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"341'6872","reported_seq":6960,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:24.923589+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:43:24.923589+0000","last_peered":"2024-09-13T14:43:24.923589+0000","last_clean":"2024-09-13T14:43:24.923589+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:43:24.923589+0000","last_undegraded":"2024-09-13T14:43:24.923589+0000","last_fullsized":"2024-09-13T14:43:24.923589+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6872,"log_dups_size":0,"ondisk_log_size":6872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6872,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":269,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657276+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:43:06.657276+0000","last_peered":"2024-09-13T14:43:06.657276+0000","last_clean":"2024-09-13T14:43:06.657276+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:43:06.657276+0000","last_undegraded":"2024-09-13T14:43:06.657276+0000","last_fullsized":"2024-09-13T14:43:06.657276+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":576,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733921+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:43:16.733921+0000","last_peered":"2024-09-13T14:43:16.733921+0000","last_clean":"2024-09-13T14:43:16.733921+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:43:16.733921+0000","last_undegraded":"2024-09-13T14:43:16.733921+0000","last_fullsized":"2024-09-13T14:43:16.733921+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"341'6812","reported_seq":6897,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:25.233221+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:43:25.233221+0000","last_peered":"2024-09-13T14:43:25.233221+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:43:25.233221+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6812,"log_dups_size":0,"ondisk_log_size":6812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304742400,"num_objects":4650,"num_object_clones":0,"num_object_copies":13950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6812,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-983}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":452,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673739+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:43:06.673739+0000","last_peered":"2024-09-13T14:43:06.673739+0000","last_clean":"2024-09-13T14:43:06.673739+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:43:06.673739+0000","last_undegraded":"2024-09-13T14:43:06.673739+0000","last_fullsized":"2024-09-13T14:43:06.673739+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":574,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902076+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:43:16.902076+0000","last_peered":"2024-09-13T14:43:16.902076+0000","last_clean":"2024-09-13T14:43:16.902076+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:43:16.902076+0000","last_undegraded":"2024-09-13T14:43:16.902076+0000","last_fullsized":"2024-09-13T14:43:16.902076+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739916+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:43:06.739916+0000","last_peered":"2024-09-13T14:43:06.739916+0000","last_clean":"2024-09-13T14:43:06.739916+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:43:06.739916+0000","last_undegraded":"2024-09-13T14:43:06.739916+0000","last_fullsized":"2024-09-13T14:43:06.739916+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"341'6552","reported_seq":6640,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:24.908631+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:43:24.908631+0000","last_peered":"2024-09-13T14:43:24.908631+0000","last_clean":"2024-09-13T14:43:24.908631+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:43:24.908631+0000","last_undegraded":"2024-09-13T14:43:24.908631+0000","last_fullsized":"2024-09-13T14:43:24.908631+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297664512,"num_objects":4542,"num_object_clones":0,"num_object_copies":13626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6552,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":156,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656816+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:43:06.656816+0000","last_peered":"2024-09-13T14:43:06.656816+0000","last_clean":"2024-09-13T14:43:06.656816+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:43:06.656816+0000","last_undegraded":"2024-09-13T14:43:06.656816+0000","last_fullsized":"2024-09-13T14:43:06.656816+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":342,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.816739+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:43:06.816739+0000","last_peered":"2024-09-13T14:43:06.816739+0000","last_clean":"2024-09-13T14:43:06.816739+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:43:06.816739+0000","last_undegraded":"2024-09-13T14:43:06.816739+0000","last_fullsized":"2024-09-13T14:43:06.816739+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":54,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740127+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:43:06.740127+0000","last_peered":"2024-09-13T14:43:06.740127+0000","last_clean":"2024-09-13T14:43:06.740127+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:43:06.740127+0000","last_undegraded":"2024-09-13T14:43:06.740127+0000","last_fullsized":"2024-09-13T14:43:06.740127+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"341'6726","reported_seq":9490,"reported_epoch":341,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:23.430887+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:43:23.430887+0000","last_peered":"2024-09-13T14:43:23.430887+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:43:23.430887+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307101696,"num_objects":4686,"num_object_clones":0,"num_object_copies":14058,"num_objects_missing_on_primary":3629,"num_objects_missing":3629,"num_objects_degraded":4004,"num_objects_misplaced":4583,"num_objects_unfound":0,"num_objects_dirty":4686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6726,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1020,"num_bytes_recovered":66846720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3629},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739579+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:43:06.739579+0000","last_peered":"2024-09-13T14:43:06.739579+0000","last_clean":"2024-09-13T14:43:06.739579+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:43:06.739579+0000","last_undegraded":"2024-09-13T14:43:06.739579+0000","last_fullsized":"2024-09-13T14:43:06.739579+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740150+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:43:06.740150+0000","last_peered":"2024-09-13T14:43:06.740150+0000","last_clean":"2024-09-13T14:43:06.740150+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:43:06.740150+0000","last_undegraded":"2024-09-13T14:43:06.740150+0000","last_fullsized":"2024-09-13T14:43:06.740150+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"341'6638","reported_seq":6726,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:24.924825+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:43:24.924825+0000","last_peered":"2024-09-13T14:43:24.924825+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:43:24.924825+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6638,"log_dups_size":0,"ondisk_log_size":6638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296222720,"num_objects":4520,"num_object_clones":0,"num_object_copies":13560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-946}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":149,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657157+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:43:06.657157+0000","last_peered":"2024-09-13T14:43:06.657157+0000","last_clean":"2024-09-13T14:43:06.657157+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:43:06.657157+0000","last_undegraded":"2024-09-13T14:43:06.657157+0000","last_fullsized":"2024-09-13T14:43:06.657157+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":566,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.909373+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:43:16.909373+0000","last_peered":"2024-09-13T14:43:16.909373+0000","last_clean":"2024-09-13T14:43:16.909373+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:43:16.909373+0000","last_undegraded":"2024-09-13T14:43:16.909373+0000","last_fullsized":"2024-09-13T14:43:16.909373+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":528,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.706167+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:43:16.706167+0000","last_peered":"2024-09-13T14:43:16.706167+0000","last_clean":"2024-09-13T14:43:16.706167+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:43:16.706167+0000","last_undegraded":"2024-09-13T14:43:16.706167+0000","last_fullsized":"2024-09-13T14:43:16.706167+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"341'6643","reported_seq":6696,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:22.830006+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:43:22.830006+0000","last_peered":"2024-09-13T14:43:22.830006+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:43:22.830006+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6643,"log_dups_size":0,"ondisk_log_size":6643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6643,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-919}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":443,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673826+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:43:06.673826+0000","last_peered":"2024-09-13T14:43:06.673826+0000","last_clean":"2024-09-13T14:43:06.673826+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:43:06.673826+0000","last_undegraded":"2024-09-13T14:43:06.673826+0000","last_fullsized":"2024-09-13T14:43:06.673826+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":865,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656967+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:43:06.656967+0000","last_peered":"2024-09-13T14:43:06.656967+0000","last_clean":"2024-09-13T14:43:06.656967+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:43:06.656967+0000","last_undegraded":"2024-09-13T14:43:06.656967+0000","last_fullsized":"2024-09-13T14:43:06.656967+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":520,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.901551+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:43:16.901551+0000","last_peered":"2024-09-13T14:43:16.901551+0000","last_clean":"2024-09-13T14:43:16.901551+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:43:16.901551+0000","last_undegraded":"2024-09-13T14:43:16.901551+0000","last_fullsized":"2024-09-13T14:43:16.901551+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.817173+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:43:06.817173+0000","last_peered":"2024-09-13T14:43:06.817173+0000","last_clean":"2024-09-13T14:43:06.817173+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:43:06.817173+0000","last_undegraded":"2024-09-13T14:43:06.817173+0000","last_fullsized":"2024-09-13T14:43:06.817173+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"341'6860","reported_seq":6946,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:24.845725+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:43:24.845725+0000","last_peered":"2024-09-13T14:43:24.845725+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:43:24.845725+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307232768,"num_objects":4688,"num_object_clones":0,"num_object_copies":14064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6860,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-992}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":374,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.708689+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:43:16.708689+0000","last_peered":"2024-09-13T14:43:16.708689+0000","last_clean":"2024-09-13T14:43:16.708689+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:43:16.708689+0000","last_undegraded":"2024-09-13T14:43:16.708689+0000","last_fullsized":"2024-09-13T14:43:16.708689+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":731,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673750+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:43:06.673750+0000","last_peered":"2024-09-13T14:43:06.673750+0000","last_clean":"2024-09-13T14:43:06.673750+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:43:06.673750+0000","last_undegraded":"2024-09-13T14:43:06.673750+0000","last_fullsized":"2024-09-13T14:43:06.673750+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":569,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733841+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:43:16.733841+0000","last_peered":"2024-09-13T14:43:16.733841+0000","last_clean":"2024-09-13T14:43:16.733841+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:43:16.733841+0000","last_undegraded":"2024-09-13T14:43:16.733841+0000","last_fullsized":"2024-09-13T14:43:16.733841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":574,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734371+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:43:16.734371+0000","last_peered":"2024-09-13T14:43:16.734371+0000","last_clean":"2024-09-13T14:43:16.734371+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:43:16.734371+0000","last_undegraded":"2024-09-13T14:43:16.734371+0000","last_fullsized":"2024-09-13T14:43:16.734371+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"341'6884","reported_seq":16513,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.573814+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:43:25.573814+0000","last_peered":"2024-09-13T14:43:25.573814+0000","last_clean":"2024-09-13T14:43:25.573814+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:43:25.573814+0000","last_undegraded":"2024-09-13T14:43:25.573814+0000","last_fullsized":"2024-09-13T14:43:25.573814+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313262080,"num_objects":4780,"num_object_clones":0,"num_object_copies":14340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6884,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":472,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.673592+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:43:06.673592+0000","last_peered":"2024-09-13T14:43:06.673592+0000","last_clean":"2024-09-13T14:43:06.673592+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:43:06.673592+0000","last_undegraded":"2024-09-13T14:43:06.673592+0000","last_fullsized":"2024-09-13T14:43:06.673592+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":537,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902035+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:43:16.902035+0000","last_peered":"2024-09-13T14:43:16.902035+0000","last_clean":"2024-09-13T14:43:16.902035+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:43:16.902035+0000","last_undegraded":"2024-09-13T14:43:16.902035+0000","last_fullsized":"2024-09-13T14:43:16.902035+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":506,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.705795+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:43:16.705795+0000","last_peered":"2024-09-13T14:43:16.705795+0000","last_clean":"2024-09-13T14:43:16.705795+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:43:16.705795+0000","last_undegraded":"2024-09-13T14:43:16.705795+0000","last_fullsized":"2024-09-13T14:43:16.705795+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657022+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:43:06.657022+0000","last_peered":"2024-09-13T14:43:06.657022+0000","last_clean":"2024-09-13T14:43:06.657022+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:43:06.657022+0000","last_undegraded":"2024-09-13T14:43:06.657022+0000","last_fullsized":"2024-09-13T14:43:06.657022+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"341'6631","reported_seq":6718,"reported_epoch":341,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:25.550580+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:43:25.550580+0000","last_peered":"2024-09-13T14:43:25.550580+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:43:25.550580+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301400064,"num_objects":4599,"num_object_clones":0,"num_object_copies":13797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-910}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":149,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656845+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:43:06.656845+0000","last_peered":"2024-09-13T14:43:06.656845+0000","last_clean":"2024-09-13T14:43:06.656845+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:43:06.656845+0000","last_undegraded":"2024-09-13T14:43:06.656845+0000","last_fullsized":"2024-09-13T14:43:06.656845+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":956,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657181+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:43:06.657181+0000","last_peered":"2024-09-13T14:43:06.657181+0000","last_clean":"2024-09-13T14:43:06.657181+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:43:06.657181+0000","last_undegraded":"2024-09-13T14:43:06.657181+0000","last_fullsized":"2024-09-13T14:43:06.657181+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739884+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:43:06.739884+0000","last_peered":"2024-09-13T14:43:06.739884+0000","last_clean":"2024-09-13T14:43:06.739884+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:43:06.739884+0000","last_undegraded":"2024-09-13T14:43:06.739884+0000","last_fullsized":"2024-09-13T14:43:06.739884+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":527,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.705905+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:43:16.705905+0000","last_peered":"2024-09-13T14:43:16.705905+0000","last_clean":"2024-09-13T14:43:16.705905+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:43:16.705905+0000","last_undegraded":"2024-09-13T14:43:16.705905+0000","last_fullsized":"2024-09-13T14:43:16.705905+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"341'6736","reported_seq":16104,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:23.392790+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:43:23.392790+0000","last_peered":"2024-09-13T14:43:23.392790+0000","last_clean":"2024-09-13T14:43:23.392790+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:43:23.392790+0000","last_undegraded":"2024-09-13T14:43:23.392790+0000","last_fullsized":"2024-09-13T14:43:23.392790+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6736,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818640+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:43:06.818640+0000","last_peered":"2024-09-13T14:43:06.818640+0000","last_clean":"2024-09-13T14:43:06.818640+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:43:06.818640+0000","last_undegraded":"2024-09-13T14:43:06.818640+0000","last_fullsized":"2024-09-13T14:43:06.818640+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":286,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657338+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:43:06.657338+0000","last_peered":"2024-09-13T14:43:06.657338+0000","last_clean":"2024-09-13T14:43:06.657338+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:43:06.657338+0000","last_undegraded":"2024-09-13T14:43:06.657338+0000","last_fullsized":"2024-09-13T14:43:06.657338+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1106,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.706826+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:43:16.706826+0000","last_peered":"2024-09-13T14:43:16.706826+0000","last_clean":"2024-09-13T14:43:16.706826+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:43:16.706826+0000","last_undegraded":"2024-09-13T14:43:16.706826+0000","last_fullsized":"2024-09-13T14:43:16.706826+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":508,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.708283+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:43:16.708283+0000","last_peered":"2024-09-13T14:43:16.708283+0000","last_clean":"2024-09-13T14:43:16.708283+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:43:16.708283+0000","last_undegraded":"2024-09-13T14:43:16.708283+0000","last_fullsized":"2024-09-13T14:43:16.708283+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"341'6692","reported_seq":6780,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.336387+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:43:25.336387+0000","last_peered":"2024-09-13T14:43:25.336387+0000","last_clean":"2024-09-13T14:43:25.336387+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:43:25.336387+0000","last_undegraded":"2024-09-13T14:43:25.336387+0000","last_fullsized":"2024-09-13T14:43:25.336387+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295698432,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6692,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.817080+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:43:06.817080+0000","last_peered":"2024-09-13T14:43:06.817080+0000","last_clean":"2024-09-13T14:43:06.817080+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:43:06.817080+0000","last_undegraded":"2024-09-13T14:43:06.817080+0000","last_fullsized":"2024-09-13T14:43:06.817080+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":517,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.705995+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:43:16.705995+0000","last_peered":"2024-09-13T14:43:16.705995+0000","last_clean":"2024-09-13T14:43:16.705995+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:43:16.705995+0000","last_undegraded":"2024-09-13T14:43:16.705995+0000","last_fullsized":"2024-09-13T14:43:16.705995+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":735,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674192+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:43:06.674192+0000","last_peered":"2024-09-13T14:43:06.674192+0000","last_clean":"2024-09-13T14:43:06.674192+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:43:06.674192+0000","last_undegraded":"2024-09-13T14:43:06.674192+0000","last_fullsized":"2024-09-13T14:43:06.674192+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":552,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733454+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:43:16.733454+0000","last_peered":"2024-09-13T14:43:16.733454+0000","last_clean":"2024-09-13T14:43:16.733454+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:43:16.733454+0000","last_undegraded":"2024-09-13T14:43:16.733454+0000","last_fullsized":"2024-09-13T14:43:16.733454+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"341'6976","reported_seq":7064,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:23.855069+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:43:23.855069+0000","last_peered":"2024-09-13T14:43:23.855069+0000","last_clean":"2024-09-13T14:43:23.855069+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:43:23.855069+0000","last_undegraded":"2024-09-13T14:43:23.855069+0000","last_fullsized":"2024-09-13T14:43:23.855069+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6976,"log_dups_size":0,"ondisk_log_size":6976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6976,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.740022+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:43:06.740022+0000","last_peered":"2024-09-13T14:43:06.740022+0000","last_clean":"2024-09-13T14:43:06.740022+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:43:06.740022+0000","last_undegraded":"2024-09-13T14:43:06.740022+0000","last_fullsized":"2024-09-13T14:43:06.740022+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":552,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902744+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:43:16.902744+0000","last_peered":"2024-09-13T14:43:16.902744+0000","last_clean":"2024-09-13T14:43:16.902744+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:43:16.902744+0000","last_undegraded":"2024-09-13T14:43:16.902744+0000","last_fullsized":"2024-09-13T14:43:16.902744+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":893,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.706750+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:43:16.706750+0000","last_peered":"2024-09-13T14:43:16.706750+0000","last_clean":"2024-09-13T14:43:16.706750+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:43:16.706750+0000","last_undegraded":"2024-09-13T14:43:16.706750+0000","last_fullsized":"2024-09-13T14:43:16.706750+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":550,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.909168+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:43:16.909168+0000","last_peered":"2024-09-13T14:43:16.909168+0000","last_clean":"2024-09-13T14:43:16.909168+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:43:16.909168+0000","last_undegraded":"2024-09-13T14:43:16.909168+0000","last_fullsized":"2024-09-13T14:43:16.909168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"341'6864","reported_seq":18216,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:22.719741+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:43:22.719741+0000","last_peered":"2024-09-13T14:43:22.719741+0000","last_clean":"2024-09-13T14:43:22.719741+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:43:22.719741+0000","last_undegraded":"2024-09-13T14:43:22.719741+0000","last_fullsized":"2024-09-13T14:43:22.719741+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6864,"log_dups_size":0,"ondisk_log_size":6864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304676888,"num_objects":4650,"num_object_clones":0,"num_object_copies":13950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4650,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6864,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":404,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.901724+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:43:16.901724+0000","last_peered":"2024-09-13T14:43:16.901724+0000","last_clean":"2024-09-13T14:43:16.901724+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:43:16.901724+0000","last_undegraded":"2024-09-13T14:43:16.901724+0000","last_fullsized":"2024-09-13T14:43:16.901724+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":588,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734212+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:43:16.734212+0000","last_peered":"2024-09-13T14:43:16.734212+0000","last_clean":"2024-09-13T14:43:16.734212+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:43:16.734212+0000","last_undegraded":"2024-09-13T14:43:16.734212+0000","last_fullsized":"2024-09-13T14:43:16.734212+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":759,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818724+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:43:06.818724+0000","last_peered":"2024-09-13T14:43:06.818724+0000","last_clean":"2024-09-13T14:43:06.818724+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:43:06.818724+0000","last_undegraded":"2024-09-13T14:43:06.818724+0000","last_fullsized":"2024-09-13T14:43:06.818724+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739650+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:43:06.739650+0000","last_peered":"2024-09-13T14:43:06.739650+0000","last_clean":"2024-09-13T14:43:06.739650+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:43:06.739650+0000","last_undegraded":"2024-09-13T14:43:06.739650+0000","last_fullsized":"2024-09-13T14:43:06.739650+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739936+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:43:06.739936+0000","last_peered":"2024-09-13T14:43:06.739936+0000","last_clean":"2024-09-13T14:43:06.739936+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:43:06.739936+0000","last_undegraded":"2024-09-13T14:43:06.739936+0000","last_fullsized":"2024-09-13T14:43:06.739936+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":50,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739892+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:43:06.739892+0000","last_peered":"2024-09-13T14:43:06.739892+0000","last_clean":"2024-09-13T14:43:06.739892+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:43:06.739892+0000","last_undegraded":"2024-09-13T14:43:06.739892+0000","last_fullsized":"2024-09-13T14:43:06.739892+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":504,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.708141+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:43:16.708141+0000","last_peered":"2024-09-13T14:43:16.708141+0000","last_clean":"2024-09-13T14:43:16.708141+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:43:16.708141+0000","last_undegraded":"2024-09-13T14:43:16.708141+0000","last_fullsized":"2024-09-13T14:43:16.708141+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.674107+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:43:06.674107+0000","last_peered":"2024-09-13T14:43:06.674107+0000","last_clean":"2024-09-13T14:43:06.674107+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:43:06.674107+0000","last_undegraded":"2024-09-13T14:43:06.674107+0000","last_fullsized":"2024-09-13T14:43:06.674107+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.656999+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:43:06.656999+0000","last_peered":"2024-09-13T14:43:06.656999+0000","last_clean":"2024-09-13T14:43:06.656999+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:43:06.656999+0000","last_undegraded":"2024-09-13T14:43:06.656999+0000","last_fullsized":"2024-09-13T14:43:06.656999+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":509,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.901750+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:43:16.901750+0000","last_peered":"2024-09-13T14:43:16.901750+0000","last_clean":"2024-09-13T14:43:16.901750+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:43:16.901750+0000","last_undegraded":"2024-09-13T14:43:16.901750+0000","last_fullsized":"2024-09-13T14:43:16.901750+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":561,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733555+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:43:16.733555+0000","last_peered":"2024-09-13T14:43:16.733555+0000","last_clean":"2024-09-13T14:43:16.733555+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:43:16.733555+0000","last_undegraded":"2024-09-13T14:43:16.733555+0000","last_fullsized":"2024-09-13T14:43:16.733555+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":565,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.909060+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:43:16.909060+0000","last_peered":"2024-09-13T14:43:16.909060+0000","last_clean":"2024-09-13T14:43:16.909060+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:43:16.909060+0000","last_undegraded":"2024-09-13T14:43:16.909060+0000","last_fullsized":"2024-09-13T14:43:16.909060+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":338,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.818681+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:43:06.818681+0000","last_peered":"2024-09-13T14:43:06.818681+0000","last_clean":"2024-09-13T14:43:06.818681+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:43:06.818681+0000","last_undegraded":"2024-09-13T14:43:06.818681+0000","last_fullsized":"2024-09-13T14:43:06.818681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":583,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.909319+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:43:16.909319+0000","last_peered":"2024-09-13T14:43:16.909319+0000","last_clean":"2024-09-13T14:43:16.909319+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:43:16.909319+0000","last_undegraded":"2024-09-13T14:43:16.909319+0000","last_fullsized":"2024-09-13T14:43:16.909319+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":552,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.734266+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:43:16.734266+0000","last_peered":"2024-09-13T14:43:16.734266+0000","last_clean":"2024-09-13T14:43:16.734266+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:43:16.734266+0000","last_undegraded":"2024-09-13T14:43:16.734266+0000","last_fullsized":"2024-09-13T14:43:16.734266+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":529,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.706323+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:43:16.706323+0000","last_peered":"2024-09-13T14:43:16.706323+0000","last_clean":"2024-09-13T14:43:16.706323+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:43:16.706323+0000","last_undegraded":"2024-09-13T14:43:16.706323+0000","last_fullsized":"2024-09-13T14:43:16.706323+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":578,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.901988+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:43:16.901988+0000","last_peered":"2024-09-13T14:43:16.901988+0000","last_clean":"2024-09-13T14:43:16.901988+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:43:16.901988+0000","last_undegraded":"2024-09-13T14:43:16.901988+0000","last_fullsized":"2024-09-13T14:43:16.901988+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":39,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.747372+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:43:06.747372+0000","last_peered":"2024-09-13T14:43:06.747372+0000","last_clean":"2024-09-13T14:43:06.747372+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:43:06.747372+0000","last_undegraded":"2024-09-13T14:43:06.747372+0000","last_fullsized":"2024-09-13T14:43:06.747372+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":475,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.707035+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:43:16.707035+0000","last_peered":"2024-09-13T14:43:16.707035+0000","last_clean":"2024-09-13T14:43:16.707035+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:43:16.707035+0000","last_undegraded":"2024-09-13T14:43:16.707035+0000","last_fullsized":"2024-09-13T14:43:16.707035+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":556,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733832+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:43:16.733832+0000","last_peered":"2024-09-13T14:43:16.733832+0000","last_clean":"2024-09-13T14:43:16.733832+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:43:16.733832+0000","last_undegraded":"2024-09-13T14:43:16.733832+0000","last_fullsized":"2024-09-13T14:43:16.733832+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739861+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:43:06.739861+0000","last_peered":"2024-09-13T14:43:06.739861+0000","last_clean":"2024-09-13T14:43:06.739861+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:43:06.739861+0000","last_undegraded":"2024-09-13T14:43:06.739861+0000","last_fullsized":"2024-09-13T14:43:06.739861+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":306,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657271+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:43:06.657271+0000","last_peered":"2024-09-13T14:43:06.657271+0000","last_clean":"2024-09-13T14:43:06.657271+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:43:06.657271+0000","last_undegraded":"2024-09-13T14:43:06.657271+0000","last_fullsized":"2024-09-13T14:43:06.657271+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":39,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.747284+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:43:06.747284+0000","last_peered":"2024-09-13T14:43:06.747284+0000","last_clean":"2024-09-13T14:43:06.747284+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:43:06.747284+0000","last_undegraded":"2024-09-13T14:43:06.747284+0000","last_fullsized":"2024-09-13T14:43:06.747284+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":272,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.657430+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:43:06.657430+0000","last_peered":"2024-09-13T14:43:06.657430+0000","last_clean":"2024-09-13T14:43:06.657430+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:43:06.657430+0000","last_undegraded":"2024-09-13T14:43:06.657430+0000","last_fullsized":"2024-09-13T14:43:06.657430+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":575,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902117+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:43:16.902117+0000","last_peered":"2024-09-13T14:43:16.902117+0000","last_clean":"2024-09-13T14:43:16.902117+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:43:16.902117+0000","last_undegraded":"2024-09-13T14:43:16.902117+0000","last_fullsized":"2024-09-13T14:43:16.902117+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":561,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.902405+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:43:16.902405+0000","last_peered":"2024-09-13T14:43:16.902405+0000","last_clean":"2024-09-13T14:43:16.902405+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:43:16.902405+0000","last_undegraded":"2024-09-13T14:43:16.902405+0000","last_fullsized":"2024-09-13T14:43:16.902405+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":40,"reported_epoch":340,"state":"active+clean","last_fresh":"2024-09-13T14:43:06.739544+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:43:06.739544+0000","last_peered":"2024-09-13T14:43:06.739544+0000","last_clean":"2024-09-13T14:43:06.739544+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:43:06.739544+0000","last_undegraded":"2024-09-13T14:43:06.739544+0000","last_fullsized":"2024-09-13T14:43:06.739544+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":566,"reported_epoch":341,"state":"active+clean","last_fresh":"2024-09-13T14:43:16.733616+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:43:16.733616+0000","last_peered":"2024-09-13T14:43:16.733616+0000","last_clean":"2024-09-13T14:43:16.733616+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:43:16.733616+0000","last_undegraded":"2024-09-13T14:43:16.733616+0000","last_fullsized":"2024-09-13T14:43:16.733616+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4868407320,"num_objects":74287,"num_object_clones":0,"num_object_copies":222861,"num_objects_missing_on_primary":3629,"num_objects_missing":3629,"num_objects_degraded":39847,"num_objects_misplaced":4583,"num_objects_unfound":0,"num_objects_dirty":74287,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108391,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21890,"num_bytes_recovered":1374158848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13339734016,"data_stored":13339721800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108391,"ondisk_log_size":108391,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397887,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2634892,"kb_used_data":1743044,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91110260,"statfs":{"total":95995035648,"available":93296906240,"internally_reserved":0,"allocated":1784877056,"data_stored":1783108270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594115,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2439992,"kb_used_data":1850096,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":91305160,"statfs":{"total":95995035648,"available":93496483840,"internally_reserved":0,"allocated":1894498304,"data_stored":1892749272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665569,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":673464,"kb_used_data":349232,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93071688,"statfs":{"total":95995035648,"available":95305408512,"internally_reserved":0,"allocated":357613568,"data_stored":355860867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986571,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1943620,"kb_used_data":1039612,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":91801532,"statfs":{"total":95995035648,"available":94004768768,"internally_reserved":0,"allocated":1064562688,"data_stored":1062802000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697131,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2877812,"kb_used_data":2151596,"kb_used_omap":4,"kb_used_meta":726203,"kb_avail":90867340,"statfs":{"total":95995035648,"available":93048156160,"internally_reserved":0,"allocated":2203234304,"data_stored":2201485756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":743631947},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860658,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2693068,"kb_used_data":2067652,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91052084,"statfs":{"total":95995035648,"available":93237334016,"internally_reserved":0,"allocated":2117275648,"data_stored":2115511599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020329,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2394608,"kb_used_data":2082728,"kb_used_omap":9,"kb_used_meta":311862,"kb_avail":91350544,"statfs":{"total":95995035648,"available":93542957056,"internally_reserved":0,"allocated":2132713472,"data_stored":2130968397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":319347525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187719,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407408,"kb_used_data":1796904,"kb_used_omap":7,"kb_used_meta":610488,"kb_avail":91337744,"statfs":{"total":95995035648,"available":93529849856,"internally_reserved":0,"allocated":1840029696,"data_stored":1838284887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":625140043},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1831272448,"data_stored":1831272448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2122186752,"data_stored":2122186752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2112557056,"data_stored":2112552984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2196176896,"data_stored":2196176896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1059454976,"data_stored":1059454976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":352456704,"data_stored":352452632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1889992704,"data_stored":1889992704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1775636480,"data_stored":1775632408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:29.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:43:29.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:29.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:29.400 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:29.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:29.619 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:29.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:29.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:29.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:29.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:29.876 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:29.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:29.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:29.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:29 smithi017 ceph-mon[103001]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:29.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:29 smithi017 ceph-mon[107556]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:30.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:29 smithi189 ceph-mon[90992]: from='client.46082 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:30.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:30.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:30.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:30.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:30.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:30.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:30.315 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:30.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:30.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:30.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:30.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:30.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:30.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:30.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:30.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:30.794 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:30.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:30.820 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:30 smithi189 ceph-mon[90992]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39812/222108 objects degraded (17.925%); 4548/222108 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:30.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:30.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:30.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:30 smithi017 ceph-mon[107556]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39812/222108 objects degraded (17.925%); 4548/222108 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:30 smithi017 ceph-mon[103001]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39812/222108 objects degraded (17.925%); 4548/222108 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:31.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:31.070 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:31.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:31.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:31.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:31.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:31.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:31.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:31.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:31.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:31.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:31.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:31.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:31.727 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:31 smithi017 ceph-mon[103001]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 39812/221163 objects degraded (18.001%); 4548/221163 objects misplaced (2.056%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:31.727 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:31 smithi017 ceph-mon[107556]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 39812/221163 objects degraded (18.001%); 4548/221163 objects misplaced (2.056%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:31.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:31.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:31.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:31.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:31.970 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:31 smithi189 ceph-mon[90992]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 39812/221163 objects degraded (18.001%); 4548/221163 objects misplaced (2.056%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:43:32.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:32.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:32.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:32.176 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:32.178 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:32.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:32.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:32.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:32.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:32.412 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:32.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:32.419 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:32.419 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:32.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:32.440 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:32.440 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:32.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:32.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:32.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:32.481 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:32.481 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:32.500 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:32.500 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:32.517 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:32.517 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:32.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:32.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:32.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:32.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:32.682 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:32.704 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:32.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:32.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:32.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:32 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3503745581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:32.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3503745581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:32.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:32.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:32 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3503745581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:33.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:33.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:33.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:33.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:33.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:33.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:33.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:33.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:33.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:33.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:33.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:33.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:33.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:33.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:33.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:33.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:33 smithi017 ceph-mon[103001]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39730/220608 objects degraded (18.009%); 4548/220608 objects misplaced (2.062%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:33.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:33 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39812/221163 objects degraded (18.001%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:33 smithi017 ceph-mon[107556]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39730/220608 objects degraded (18.009%); 4548/220608 objects misplaced (2.062%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:33 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39812/221163 objects degraded (18.001%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:33.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:33.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:34.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:33 smithi189 ceph-mon[90992]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 39730/220608 objects degraded (18.009%); 4548/220608 objects misplaced (2.062%); 953 KiB/s, 14 objects/s recovering 2024-09-13T14:43:34.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:33 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39812/221163 objects degraded (18.001%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:34.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:34.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:34.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:34.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:34.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:34.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:34.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:34.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:34.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:34.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:34.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:34.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:34.720 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:43:34.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:34.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:34.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:34.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:35.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:35.147 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:35.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:35.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:35.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:35.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:35.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:35.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:35.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:35.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:35.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:35.695 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:35.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:35.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:35.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:35.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.978 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:35.978 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:35.978 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[103001]: osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:43:35.978 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[103001]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39696/219627 objects degraded (18.074%); 4513/219627 objects misplaced (2.055%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:35.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:35.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:35.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[107556]: osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:43:35.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:35 smithi017 ceph-mon[107556]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39696/219627 objects degraded (18.074%); 4513/219627 objects misplaced (2.055%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:35 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1789476743' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:35 smithi189 ceph-mon[90992]: osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:43:36.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:35 smithi189 ceph-mon[90992]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39696/219627 objects degraded (18.074%); 4513/219627 objects misplaced (2.055%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:36.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:36.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:36.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:36.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:36.203 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:36.203 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:36.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:36.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:36.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:36.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:36.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:36.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:36.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:36.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:36.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:36.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:36.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:36.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:36.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:36.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:43:37.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:37.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:37.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:37.195 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:37.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.445 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.547 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.635 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:37] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:43:37.670 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:37.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:37.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:43:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:37.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:37.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:37.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:37.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:38.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:38.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:38.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:38.198 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:38.287 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:38.289 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:38.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:38.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:38.412 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:38 smithi017 ceph-mon[103001]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 39602/218940 objects degraded (18.088%); 4513/218940 objects misplaced (2.061%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:38.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:38 smithi017 ceph-mon[107556]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 39602/218940 objects degraded (18.088%); 4513/218940 objects misplaced (2.061%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:38.414 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:38.414 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 39602/218940 objects degraded (18.088%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1415,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":343,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":72980,"data_bytes":4785605302,"bytes_used":18250780672,"bytes_avail":749709504512,"bytes_total":767960285184,"degraded_objects":39602,"degraded_total":218940,"degraded_ratio":0.18088060655887456,"misplaced_objects":4513,"misplaced_total":218940,"misplaced_ratio":0.020612953320544442,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":1035196,"recovering_keys_per_sec":0,"num_objects_recovered":158,"num_bytes_recovered":10354688,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":133},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 20s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:43:38.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:38.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:38.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:38.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:38 smithi189 ceph-mon[90992]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 39602/218940 objects degraded (18.088%); 4513/218940 objects misplaced (2.061%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:43:38.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:38.577 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:38.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:38.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:38.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:38.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:38.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:38.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:39.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:39.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:39.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:39.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:39.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:39.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:39.448 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:43:39.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:43:39.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3615489759' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:39.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39602/218940 objects degraded (18.088%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:39.545 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3615489759' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:39.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39602/218940 objects degraded (18.088%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:39.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3615489759' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:43:39.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39602/218940 objects degraded (18.088%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:39.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:39.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:39.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:39.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:39.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:39.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:39.927 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:39.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:40.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:40.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:40.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:40.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:40.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:40.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:40 smithi189 ceph-mon[90992]: pgmap v1028: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39564/218136 objects degraded (18.137%); 4474/218136 objects misplaced (2.051%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:40.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:40.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:40.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:40 smithi017 ceph-mon[103001]: pgmap v1028: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39564/218136 objects degraded (18.137%); 4474/218136 objects misplaced (2.051%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:40 smithi017 ceph-mon[107556]: pgmap v1028: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39564/218136 objects degraded (18.137%); 4474/218136 objects misplaced (2.051%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:40.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:40.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:40.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:40.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:40.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:41.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:41.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:41.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:41.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:41.273 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:41.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:41.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:41.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:41.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:41.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:41.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:41.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:41.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:41.688 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:41.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:41.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:41.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:41.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:41.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:42.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:42.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:42.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:42.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:42.262 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:42.262 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:42.273 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:42.273 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:42.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:42.291 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:42.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:42.310 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:42.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:42.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:42.327 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:42.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:42.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:42.347 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:42.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:42.365 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:42.381 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:42.382 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:42.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:42.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:42 smithi189 ceph-mon[90992]: pgmap v1029: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 39564/217218 objects degraded (18.214%); 4474/217218 objects misplaced (2.060%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:42.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/286458215' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[103001]: pgmap v1029: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 39564/217218 objects degraded (18.214%); 4474/217218 objects misplaced (2.060%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/286458215' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[107556]: pgmap v1029: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 39564/217218 objects degraded (18.214%); 4474/217218 objects misplaced (2.060%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:42.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:42.516 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/286458215' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:42.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:42.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:42.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:42.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:42.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:42.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:42.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:42.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:43.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:43.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:43.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:43.266 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:43.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:43.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:43.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:43.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:43.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:43.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:43.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:43.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:43.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:43.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:44.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:44.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:44.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:44.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:44.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:44.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:44.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:44 smithi017 ceph-mon[103001]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 39496/216684 objects degraded (18.227%); 4474/216684 objects misplaced (2.065%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:43:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39496/216684 objects degraded (18.227%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:44 smithi017 ceph-mon[107556]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 39496/216684 objects degraded (18.227%); 4474/216684 objects misplaced (2.065%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:43:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39496/216684 objects degraded (18.227%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:44.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:44.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:44.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:44.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:44.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:44.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:44 smithi189 ceph-mon[90992]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 39496/216684 objects degraded (18.227%); 4474/216684 objects misplaced (2.065%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:43:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39496/216684 objects degraded (18.227%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:44.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:44.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:44.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:44.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:45.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:45.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:45.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:45.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:45.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:45.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:45.318 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:43:45.318 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:43:45.318 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:43:45.318 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:43:45.318 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:43:45.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:45.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:45.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:45.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:45.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:46.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:46.220 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:46.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:46.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:46.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:46 smithi017 ceph-mon[103001]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 39459/215850 objects degraded (18.281%); 4436/215850 objects misplaced (2.055%); 965 KiB/s, 15 objects/s recovering 2024-09-13T14:43:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:46 smithi017 ceph-mon[107556]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 39459/215850 objects degraded (18.281%); 4436/215850 objects misplaced (2.055%); 965 KiB/s, 15 objects/s recovering 2024-09-13T14:43:46.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:46.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:46.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:43:46.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:46.697 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:46 smithi189 ceph-mon[90992]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 39459/215850 objects degraded (18.281%); 4436/215850 objects misplaced (2.055%); 965 KiB/s, 15 objects/s recovering 2024-09-13T14:43:46.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:46.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:46.944 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:46.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.070 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:47.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:47.212 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:47.263 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:43:47.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:47.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:47.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:47.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:47.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:47.615 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:47] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-13T14:43:47.615 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:47.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:47.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:47.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:47.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:47.930 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:43:47.931 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:43:47.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:47.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:48.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:48.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:48.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:48.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:48.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:48.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:48.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:48.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[107556]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 39363/215229 objects degraded (18.289%); 4436/215229 objects misplaced (2.061%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[107556]: osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[103001]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 39363/215229 objects degraded (18.289%); 4436/215229 objects misplaced (2.061%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:43:48.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:48.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:48.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:48 smithi017 ceph-mon[103001]: osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:43:48.564 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:48.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:48.688 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:48.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:48.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:48 smithi189 ceph-mon[90992]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 39363/215229 objects degraded (18.289%); 4436/215229 objects misplaced (2.061%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:43:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:48 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/92719666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:48 smithi189 ceph-mon[90992]: osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:43:48.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:48.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:48.942 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:48.968 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:49.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:49.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:49.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:49.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:49 smithi017 ceph-mon[103001]: from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39363/215229 objects degraded (18.289%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:49 smithi017 ceph-mon[107556]: from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39363/215229 objects degraded (18.289%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:49.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:49.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:49.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:49.589 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:49.669 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:49.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:49 smithi189 ceph-mon[90992]: from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39363/215229 objects degraded (18.289%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:49.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:49.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:49.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:49.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:50.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:50.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:50.109 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1033,"stamp":"2024-09-13T14:43:47.262826+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4704537270,"num_objects":71743,"num_object_clones":0,"num_object_copies":215229,"num_objects_missing_on_primary":3485,"num_objects_missing":3485,"num_objects_degraded":39363,"num_objects_misplaced":4436,"num_objects_unfound":0,"num_objects_dirty":71743,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":111910,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22264,"num_bytes_recovered":1395099805,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113095,"ondisk_log_size":113095,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17599504,"kb_used_data":12644624,"kb_used_omap":44,"kb_used_meta":4954771,"kb_avail":732361712,"statfs":{"total":767960285184,"available":749938393088,"internally_reserved":0,"allocated":12948094976,"data_stored":12933943583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5073685751},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81068032,"num_objects":-1237,"num_object_clones":0,"num_object_copies":-3711,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-239,"num_objects_misplaced":-77,"num_objects_unfound":0,"num_objects_dirty":-1237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1237,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.070207"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":569,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.723132+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:43:34.723132+0000","last_peered":"2024-09-13T14:43:34.723132+0000","last_clean":"2024-09-13T14:43:34.723132+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:43:34.723132+0000","last_undegraded":"2024-09-13T14:43:34.723132+0000","last_fullsized":"2024-09-13T14:43:34.723132+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":341,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.855275+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:43:25.855275+0000","last_peered":"2024-09-13T14:43:25.855275+0000","last_clean":"2024-09-13T14:43:25.855275+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:43:25.855275+0000","last_undegraded":"2024-09-13T14:43:25.855275+0000","last_fullsized":"2024-09-13T14:43:25.855275+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":270,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701731+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:43:25.701731+0000","last_peered":"2024-09-13T14:43:25.701731+0000","last_clean":"2024-09-13T14:43:25.701731+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:43:25.701731+0000","last_undegraded":"2024-09-13T14:43:25.701731+0000","last_fullsized":"2024-09-13T14:43:25.701731+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":554,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749596+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:43:34.749596+0000","last_peered":"2024-09-13T14:43:34.749596+0000","last_clean":"2024-09-13T14:43:34.749596+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:43:34.749596+0000","last_undegraded":"2024-09-13T14:43:34.749596+0000","last_fullsized":"2024-09-13T14:43:34.749596+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":555,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.750039+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:43:34.750039+0000","last_peered":"2024-09-13T14:43:34.750039+0000","last_clean":"2024-09-13T14:43:34.750039+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:43:34.750039+0000","last_undegraded":"2024-09-13T14:43:34.750039+0000","last_fullsized":"2024-09-13T14:43:34.750039+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":310,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.855027+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:43:25.855027+0000","last_peered":"2024-09-13T14:43:25.855027+0000","last_clean":"2024-09-13T14:43:25.855027+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:43:25.855027+0000","last_undegraded":"2024-09-13T14:43:25.855027+0000","last_fullsized":"2024-09-13T14:43:25.855027+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":569,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.723813+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:43:34.723813+0000","last_peered":"2024-09-13T14:43:34.723813+0000","last_clean":"2024-09-13T14:43:34.723813+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:43:34.723813+0000","last_undegraded":"2024-09-13T14:43:34.723813+0000","last_fullsized":"2024-09-13T14:43:34.723813+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":332,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.849650+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:43:25.849650+0000","last_peered":"2024-09-13T14:43:25.849650+0000","last_clean":"2024-09-13T14:43:25.849650+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:43:25.849650+0000","last_undegraded":"2024-09-13T14:43:25.849650+0000","last_fullsized":"2024-09-13T14:43:25.849650+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"343'6887","reported_seq":10702,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:47.025072+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:43:47.025072+0000","last_peered":"2024-09-13T14:43:47.025072+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:43:47.025072+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288555008,"num_objects":4403,"num_object_clones":0,"num_object_copies":13209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7719,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6887,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1243,"num_bytes_recovered":81461248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3418}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":336,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.855376+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:43:25.855376+0000","last_peered":"2024-09-13T14:43:25.855376+0000","last_clean":"2024-09-13T14:43:25.855376+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:43:25.855376+0000","last_undegraded":"2024-09-13T14:43:25.855376+0000","last_fullsized":"2024-09-13T14:43:25.855376+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701054+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:43:25.701054+0000","last_peered":"2024-09-13T14:43:25.701054+0000","last_clean":"2024-09-13T14:43:25.701054+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:43:25.701054+0000","last_undegraded":"2024-09-13T14:43:25.701054+0000","last_fullsized":"2024-09-13T14:43:25.701054+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"343'6965","reported_seq":16505,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:42.712164+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:43:42.712164+0000","last_peered":"2024-09-13T14:43:42.712164+0000","last_clean":"2024-09-13T14:43:42.712164+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:43:42.712164+0000","last_undegraded":"2024-09-13T14:43:42.712164+0000","last_fullsized":"2024-09-13T14:43:42.712164+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6965,"log_dups_size":0,"ondisk_log_size":6965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296419328,"num_objects":4523,"num_object_clones":0,"num_object_copies":13569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6965,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":354,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.855779+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:43:25.855779+0000","last_peered":"2024-09-13T14:43:25.855779+0000","last_clean":"2024-09-13T14:43:25.855779+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:43:25.855779+0000","last_undegraded":"2024-09-13T14:43:25.855779+0000","last_fullsized":"2024-09-13T14:43:25.855779+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.735903+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:43:25.735903+0000","last_peered":"2024-09-13T14:43:25.735903+0000","last_clean":"2024-09-13T14:43:25.735903+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:43:25.735903+0000","last_undegraded":"2024-09-13T14:43:25.735903+0000","last_fullsized":"2024-09-13T14:43:25.735903+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"343'7151","reported_seq":7243,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:43.976364+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:43:43.976364+0000","last_peered":"2024-09-13T14:43:43.976364+0000","last_clean":"2024-09-13T14:43:43.976364+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:43:43.976364+0000","last_undegraded":"2024-09-13T14:43:43.976364+0000","last_fullsized":"2024-09-13T14:43:43.976364+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304545792,"num_objects":4647,"num_object_clones":0,"num_object_copies":13941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7151,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":308,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.857421+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:43:25.857421+0000","last_peered":"2024-09-13T14:43:25.857421+0000","last_clean":"2024-09-13T14:43:25.857421+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:43:25.857421+0000","last_undegraded":"2024-09-13T14:43:25.857421+0000","last_fullsized":"2024-09-13T14:43:25.857421+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":323,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.849976+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:43:25.849976+0000","last_peered":"2024-09-13T14:43:25.849976+0000","last_clean":"2024-09-13T14:43:25.849976+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:43:25.849976+0000","last_undegraded":"2024-09-13T14:43:25.849976+0000","last_fullsized":"2024-09-13T14:43:25.849976+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"343'7022","reported_seq":7114,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:44.604655+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:43:44.604655+0000","last_peered":"2024-09-13T14:43:44.604655+0000","last_clean":"2024-09-13T14:43:44.604655+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:43:44.604655+0000","last_undegraded":"2024-09-13T14:43:44.604655+0000","last_fullsized":"2024-09-13T14:43:44.604655+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296484864,"num_objects":4524,"num_object_clones":0,"num_object_copies":13572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7022,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":271,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701392+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:43:25.701392+0000","last_peered":"2024-09-13T14:43:25.701392+0000","last_clean":"2024-09-13T14:43:25.701392+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:43:25.701392+0000","last_undegraded":"2024-09-13T14:43:25.701392+0000","last_fullsized":"2024-09-13T14:43:25.701392+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":580,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749957+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:43:34.749957+0000","last_peered":"2024-09-13T14:43:34.749957+0000","last_clean":"2024-09-13T14:43:34.749957+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:43:34.749957+0000","last_undegraded":"2024-09-13T14:43:34.749957+0000","last_fullsized":"2024-09-13T14:43:34.749957+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"343'6969","reported_seq":7059,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:45.271120+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:43:45.271120+0000","last_peered":"2024-09-13T14:43:45.271120+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:43:45.271120+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6969,"log_dups_size":0,"ondisk_log_size":6969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294453248,"num_objects":4493,"num_object_clones":0,"num_object_copies":13479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6969,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1140}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":454,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.854851+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:43:25.854851+0000","last_peered":"2024-09-13T14:43:25.854851+0000","last_clean":"2024-09-13T14:43:25.854851+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:43:25.854851+0000","last_undegraded":"2024-09-13T14:43:25.854851+0000","last_fullsized":"2024-09-13T14:43:25.854851+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":578,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.723507+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:43:34.723507+0000","last_peered":"2024-09-13T14:43:34.723507+0000","last_clean":"2024-09-13T14:43:34.723507+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:43:34.723507+0000","last_undegraded":"2024-09-13T14:43:34.723507+0000","last_fullsized":"2024-09-13T14:43:34.723507+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.735961+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:43:25.735961+0000","last_peered":"2024-09-13T14:43:25.735961+0000","last_clean":"2024-09-13T14:43:25.735961+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:43:25.735961+0000","last_undegraded":"2024-09-13T14:43:25.735961+0000","last_fullsized":"2024-09-13T14:43:25.735961+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"343'6703","reported_seq":6796,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:44.693453+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:43:44.693453+0000","last_peered":"2024-09-13T14:43:44.693453+0000","last_clean":"2024-09-13T14:43:44.693453+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:43:44.693453+0000","last_undegraded":"2024-09-13T14:43:44.693453+0000","last_fullsized":"2024-09-13T14:43:44.693453+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287768576,"num_objects":4391,"num_object_clones":0,"num_object_copies":13173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6703,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":158,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701070+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:43:25.701070+0000","last_peered":"2024-09-13T14:43:25.701070+0000","last_clean":"2024-09-13T14:43:25.701070+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:43:25.701070+0000","last_undegraded":"2024-09-13T14:43:25.701070+0000","last_fullsized":"2024-09-13T14:43:25.701070+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":344,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.849851+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:43:25.849851+0000","last_peered":"2024-09-13T14:43:25.849851+0000","last_clean":"2024-09-13T14:43:25.849851+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:43:25.849851+0000","last_undegraded":"2024-09-13T14:43:25.849851+0000","last_fullsized":"2024-09-13T14:43:25.849851+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":56,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736347+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:43:25.736347+0000","last_peered":"2024-09-13T14:43:25.736347+0000","last_clean":"2024-09-13T14:43:25.736347+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:43:25.736347+0000","last_undegraded":"2024-09-13T14:43:25.736347+0000","last_fullsized":"2024-09-13T14:43:25.736347+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"343'6873","reported_seq":10076,"reported_epoch":343,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:43.621866+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:43:43.621866+0000","last_peered":"2024-09-13T14:43:43.621866+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:43:43.621866+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":3485,"num_objects_missing":3485,"num_objects_degraded":3860,"num_objects_misplaced":4436,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6873,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1167,"num_bytes_recovered":76480512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3485},{"shards":"0,3,4","objects":951},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.735788+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:43:25.735788+0000","last_peered":"2024-09-13T14:43:25.735788+0000","last_clean":"2024-09-13T14:43:25.735788+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:43:25.735788+0000","last_undegraded":"2024-09-13T14:43:25.735788+0000","last_fullsized":"2024-09-13T14:43:25.735788+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736396+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:43:25.736396+0000","last_peered":"2024-09-13T14:43:25.736396+0000","last_clean":"2024-09-13T14:43:25.736396+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:43:25.736396+0000","last_undegraded":"2024-09-13T14:43:25.736396+0000","last_fullsized":"2024-09-13T14:43:25.736396+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"343'6771","reported_seq":6864,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:44.036873+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:43:44.036873+0000","last_peered":"2024-09-13T14:43:44.036873+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:43:44.036873+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6771,"log_dups_size":0,"ondisk_log_size":6771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287506432,"num_objects":4387,"num_object_clones":0,"num_object_copies":13161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6771,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1079}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":151,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701449+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:43:25.701449+0000","last_peered":"2024-09-13T14:43:25.701449+0000","last_clean":"2024-09-13T14:43:25.701449+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:43:25.701449+0000","last_undegraded":"2024-09-13T14:43:25.701449+0000","last_fullsized":"2024-09-13T14:43:25.701449+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":571,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.731458+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:43:34.731458+0000","last_peered":"2024-09-13T14:43:34.731458+0000","last_clean":"2024-09-13T14:43:34.731458+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:43:34.731458+0000","last_undegraded":"2024-09-13T14:43:34.731458+0000","last_fullsized":"2024-09-13T14:43:34.731458+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":533,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.732560+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:43:34.732560+0000","last_peered":"2024-09-13T14:43:34.732560+0000","last_clean":"2024-09-13T14:43:34.732560+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:43:34.732560+0000","last_undegraded":"2024-09-13T14:43:34.732560+0000","last_fullsized":"2024-09-13T14:43:34.732560+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"343'6803","reported_seq":6858,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:42.900626+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:43:42.900626+0000","last_peered":"2024-09-13T14:43:42.900626+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:43:42.900626+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6803,"log_dups_size":0,"ondisk_log_size":6803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289472512,"num_objects":4417,"num_object_clones":0,"num_object_copies":13251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6803,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1079}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":445,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.854963+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:43:25.854963+0000","last_peered":"2024-09-13T14:43:25.854963+0000","last_clean":"2024-09-13T14:43:25.854963+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:43:25.854963+0000","last_undegraded":"2024-09-13T14:43:25.854963+0000","last_fullsized":"2024-09-13T14:43:25.854963+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":867,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701280+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:43:25.701280+0000","last_peered":"2024-09-13T14:43:25.701280+0000","last_clean":"2024-09-13T14:43:25.701280+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:43:25.701280+0000","last_undegraded":"2024-09-13T14:43:25.701280+0000","last_fullsized":"2024-09-13T14:43:25.701280+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":524,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.722171+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:43:34.722171+0000","last_peered":"2024-09-13T14:43:34.722171+0000","last_clean":"2024-09-13T14:43:34.722171+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:43:34.722171+0000","last_undegraded":"2024-09-13T14:43:34.722171+0000","last_fullsized":"2024-09-13T14:43:34.722171+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":309,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.850291+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:43:25.850291+0000","last_peered":"2024-09-13T14:43:25.850291+0000","last_clean":"2024-09-13T14:43:25.850291+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:43:25.850291+0000","last_undegraded":"2024-09-13T14:43:25.850291+0000","last_fullsized":"2024-09-13T14:43:25.850291+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"343'7030","reported_seq":7120,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:44.473279+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:43:44.473279+0000","last_peered":"2024-09-13T14:43:44.473279+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:43:44.473279+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296091648,"num_objects":4518,"num_object_clones":0,"num_object_copies":13554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7030,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1162}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":378,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.732701+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:43:34.732701+0000","last_peered":"2024-09-13T14:43:34.732701+0000","last_clean":"2024-09-13T14:43:34.732701+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:43:34.732701+0000","last_undegraded":"2024-09-13T14:43:34.732701+0000","last_fullsized":"2024-09-13T14:43:34.732701+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":733,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.854888+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:43:25.854888+0000","last_peered":"2024-09-13T14:43:25.854888+0000","last_clean":"2024-09-13T14:43:25.854888+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:43:25.854888+0000","last_undegraded":"2024-09-13T14:43:25.854888+0000","last_fullsized":"2024-09-13T14:43:25.854888+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":573,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749667+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:43:34.749667+0000","last_peered":"2024-09-13T14:43:34.749667+0000","last_clean":"2024-09-13T14:43:34.749667+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:43:34.749667+0000","last_undegraded":"2024-09-13T14:43:34.749667+0000","last_fullsized":"2024-09-13T14:43:34.749667+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":578,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.750199+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:43:34.750199+0000","last_peered":"2024-09-13T14:43:34.750199+0000","last_clean":"2024-09-13T14:43:34.750199+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:43:34.750199+0000","last_undegraded":"2024-09-13T14:43:34.750199+0000","last_fullsized":"2024-09-13T14:43:34.750199+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"343'7056","reported_seq":16689,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:45.534183+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:43:45.534183+0000","last_peered":"2024-09-13T14:43:45.534183+0000","last_clean":"2024-09-13T14:43:45.534183+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:43:45.534183+0000","last_undegraded":"2024-09-13T14:43:45.534183+0000","last_fullsized":"2024-09-13T14:43:45.534183+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7056,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":474,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.855067+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:43:25.855067+0000","last_peered":"2024-09-13T14:43:25.855067+0000","last_clean":"2024-09-13T14:43:25.855067+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:43:25.855067+0000","last_undegraded":"2024-09-13T14:43:25.855067+0000","last_fullsized":"2024-09-13T14:43:25.855067+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":542,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.721952+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:43:34.721952+0000","last_peered":"2024-09-13T14:43:34.721952+0000","last_clean":"2024-09-13T14:43:34.721952+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:43:34.721952+0000","last_undegraded":"2024-09-13T14:43:34.721952+0000","last_fullsized":"2024-09-13T14:43:34.721952+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":510,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.729319+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:43:34.729319+0000","last_peered":"2024-09-13T14:43:34.729319+0000","last_clean":"2024-09-13T14:43:34.729319+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:43:34.729319+0000","last_undegraded":"2024-09-13T14:43:34.729319+0000","last_fullsized":"2024-09-13T14:43:34.729319+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":92,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701617+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:43:25.701617+0000","last_peered":"2024-09-13T14:43:25.701617+0000","last_clean":"2024-09-13T14:43:25.701617+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:43:25.701617+0000","last_undegraded":"2024-09-13T14:43:25.701617+0000","last_fullsized":"2024-09-13T14:43:25.701617+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"343'6800","reported_seq":6891,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:45.584059+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:43:45.584059+0000","last_peered":"2024-09-13T14:43:45.584059+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:43:45.584059+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6800,"log_dups_size":0,"ondisk_log_size":6800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6800,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1079}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":151,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701178+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:43:25.701178+0000","last_peered":"2024-09-13T14:43:25.701178+0000","last_clean":"2024-09-13T14:43:25.701178+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:43:25.701178+0000","last_undegraded":"2024-09-13T14:43:25.701178+0000","last_fullsized":"2024-09-13T14:43:25.701178+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":958,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701517+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:43:25.701517+0000","last_peered":"2024-09-13T14:43:25.701517+0000","last_clean":"2024-09-13T14:43:25.701517+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:43:25.701517+0000","last_undegraded":"2024-09-13T14:43:25.701517+0000","last_fullsized":"2024-09-13T14:43:25.701517+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736210+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:43:25.736210+0000","last_peered":"2024-09-13T14:43:25.736210+0000","last_clean":"2024-09-13T14:43:25.736210+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:43:25.736210+0000","last_undegraded":"2024-09-13T14:43:25.736210+0000","last_fullsized":"2024-09-13T14:43:25.736210+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":531,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.728712+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:43:34.728712+0000","last_peered":"2024-09-13T14:43:34.728712+0000","last_clean":"2024-09-13T14:43:34.728712+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:43:34.728712+0000","last_undegraded":"2024-09-13T14:43:34.728712+0000","last_fullsized":"2024-09-13T14:43:34.728712+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"343'6898","reported_seq":16270,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:43.947986+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:43:43.947986+0000","last_peered":"2024-09-13T14:43:43.947986+0000","last_clean":"2024-09-13T14:43:43.947986+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:43:43.947986+0000","last_undegraded":"2024-09-13T14:43:43.947986+0000","last_fullsized":"2024-09-13T14:43:43.947986+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292421632,"num_objects":4462,"num_object_clones":0,"num_object_copies":13386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6898,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":342,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.850163+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:43:25.850163+0000","last_peered":"2024-09-13T14:43:25.850163+0000","last_clean":"2024-09-13T14:43:25.850163+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:43:25.850163+0000","last_undegraded":"2024-09-13T14:43:25.850163+0000","last_fullsized":"2024-09-13T14:43:25.850163+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":288,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701628+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:43:25.701628+0000","last_peered":"2024-09-13T14:43:25.701628+0000","last_clean":"2024-09-13T14:43:25.701628+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:43:25.701628+0000","last_undegraded":"2024-09-13T14:43:25.701628+0000","last_fullsized":"2024-09-13T14:43:25.701628+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1110,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.728852+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:43:34.728852+0000","last_peered":"2024-09-13T14:43:34.728852+0000","last_clean":"2024-09-13T14:43:34.728852+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:43:34.728852+0000","last_undegraded":"2024-09-13T14:43:34.728852+0000","last_fullsized":"2024-09-13T14:43:34.728852+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":512,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.732515+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:43:34.732515+0000","last_peered":"2024-09-13T14:43:34.732515+0000","last_clean":"2024-09-13T14:43:34.732515+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:43:34.732515+0000","last_undegraded":"2024-09-13T14:43:34.732515+0000","last_fullsized":"2024-09-13T14:43:34.732515+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"343'6850","reported_seq":6942,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:45.500083+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:43:45.500083+0000","last_peered":"2024-09-13T14:43:45.500083+0000","last_clean":"2024-09-13T14:43:45.500083+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:43:45.500083+0000","last_undegraded":"2024-09-13T14:43:45.500083+0000","last_fullsized":"2024-09-13T14:43:45.500083+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285343744,"num_objects":4354,"num_object_clones":0,"num_object_copies":13062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6850,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":342,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.850209+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:43:25.850209+0000","last_peered":"2024-09-13T14:43:25.850209+0000","last_clean":"2024-09-13T14:43:25.850209+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:43:25.850209+0000","last_undegraded":"2024-09-13T14:43:25.850209+0000","last_fullsized":"2024-09-13T14:43:25.850209+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":521,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.728782+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:43:34.728782+0000","last_peered":"2024-09-13T14:43:34.728782+0000","last_clean":"2024-09-13T14:43:34.728782+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:43:34.728782+0000","last_undegraded":"2024-09-13T14:43:34.728782+0000","last_fullsized":"2024-09-13T14:43:34.728782+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":737,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.857491+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:43:25.857491+0000","last_peered":"2024-09-13T14:43:25.857491+0000","last_clean":"2024-09-13T14:43:25.857491+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:43:25.857491+0000","last_undegraded":"2024-09-13T14:43:25.857491+0000","last_fullsized":"2024-09-13T14:43:25.857491+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":556,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749873+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:43:34.749873+0000","last_peered":"2024-09-13T14:43:34.749873+0000","last_clean":"2024-09-13T14:43:34.749873+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:43:34.749873+0000","last_undegraded":"2024-09-13T14:43:34.749873+0000","last_fullsized":"2024-09-13T14:43:34.749873+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"343'7149","reported_seq":7241,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:43.784222+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:43:43.784222+0000","last_peered":"2024-09-13T14:43:43.784222+0000","last_clean":"2024-09-13T14:43:43.784222+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:43:43.784222+0000","last_undegraded":"2024-09-13T14:43:43.784222+0000","last_fullsized":"2024-09-13T14:43:43.784222+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297598976,"num_objects":4541,"num_object_clones":0,"num_object_copies":13623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7149,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736277+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:43:25.736277+0000","last_peered":"2024-09-13T14:43:25.736277+0000","last_clean":"2024-09-13T14:43:25.736277+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:43:25.736277+0000","last_undegraded":"2024-09-13T14:43:25.736277+0000","last_fullsized":"2024-09-13T14:43:25.736277+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":557,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.724066+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:43:34.724066+0000","last_peered":"2024-09-13T14:43:34.724066+0000","last_clean":"2024-09-13T14:43:34.724066+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:43:34.724066+0000","last_undegraded":"2024-09-13T14:43:34.724066+0000","last_fullsized":"2024-09-13T14:43:34.724066+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":897,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.728569+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:43:34.728569+0000","last_peered":"2024-09-13T14:43:34.728569+0000","last_clean":"2024-09-13T14:43:34.728569+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:43:34.728569+0000","last_undegraded":"2024-09-13T14:43:34.728569+0000","last_fullsized":"2024-09-13T14:43:34.728569+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":554,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.731276+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:43:34.731276+0000","last_peered":"2024-09-13T14:43:34.731276+0000","last_clean":"2024-09-13T14:43:34.731276+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:43:34.731276+0000","last_undegraded":"2024-09-13T14:43:34.731276+0000","last_fullsized":"2024-09-13T14:43:34.731276+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"343'7036","reported_seq":18390,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:42.650736+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:43:42.650736+0000","last_peered":"2024-09-13T14:43:42.650736+0000","last_clean":"2024-09-13T14:43:42.650736+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:43:42.650736+0000","last_undegraded":"2024-09-13T14:43:42.650736+0000","last_fullsized":"2024-09-13T14:43:42.650736+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293404696,"num_objects":4478,"num_object_clones":0,"num_object_copies":13434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4478,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7036,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":408,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.722993+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:43:34.722993+0000","last_peered":"2024-09-13T14:43:34.722993+0000","last_clean":"2024-09-13T14:43:34.722993+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:43:34.722993+0000","last_undegraded":"2024-09-13T14:43:34.722993+0000","last_fullsized":"2024-09-13T14:43:34.722993+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":592,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749931+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:43:34.749931+0000","last_peered":"2024-09-13T14:43:34.749931+0000","last_clean":"2024-09-13T14:43:34.749931+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:43:34.749931+0000","last_undegraded":"2024-09-13T14:43:34.749931+0000","last_fullsized":"2024-09-13T14:43:34.749931+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":761,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.850305+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:43:25.850305+0000","last_peered":"2024-09-13T14:43:25.850305+0000","last_clean":"2024-09-13T14:43:25.850305+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:43:25.850305+0000","last_undegraded":"2024-09-13T14:43:25.850305+0000","last_fullsized":"2024-09-13T14:43:25.850305+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.735644+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:43:25.735644+0000","last_peered":"2024-09-13T14:43:25.735644+0000","last_clean":"2024-09-13T14:43:25.735644+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:43:25.735644+0000","last_undegraded":"2024-09-13T14:43:25.735644+0000","last_fullsized":"2024-09-13T14:43:25.735644+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736288+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:43:25.736288+0000","last_peered":"2024-09-13T14:43:25.736288+0000","last_clean":"2024-09-13T14:43:25.736288+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:43:25.736288+0000","last_undegraded":"2024-09-13T14:43:25.736288+0000","last_fullsized":"2024-09-13T14:43:25.736288+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":52,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736001+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:43:25.736001+0000","last_peered":"2024-09-13T14:43:25.736001+0000","last_clean":"2024-09-13T14:43:25.736001+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:43:25.736001+0000","last_undegraded":"2024-09-13T14:43:25.736001+0000","last_fullsized":"2024-09-13T14:43:25.736001+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":508,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.732374+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:43:34.732374+0000","last_peered":"2024-09-13T14:43:34.732374+0000","last_clean":"2024-09-13T14:43:34.732374+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:43:34.732374+0000","last_undegraded":"2024-09-13T14:43:34.732374+0000","last_fullsized":"2024-09-13T14:43:34.732374+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":342,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.857305+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:43:25.857305+0000","last_peered":"2024-09-13T14:43:25.857305+0000","last_clean":"2024-09-13T14:43:25.857305+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:43:25.857305+0000","last_undegraded":"2024-09-13T14:43:25.857305+0000","last_fullsized":"2024-09-13T14:43:25.857305+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":308,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701627+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:43:25.701627+0000","last_peered":"2024-09-13T14:43:25.701627+0000","last_clean":"2024-09-13T14:43:25.701627+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:43:25.701627+0000","last_undegraded":"2024-09-13T14:43:25.701627+0000","last_fullsized":"2024-09-13T14:43:25.701627+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":513,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.721642+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:43:34.721642+0000","last_peered":"2024-09-13T14:43:34.721642+0000","last_clean":"2024-09-13T14:43:34.721642+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:43:34.721642+0000","last_undegraded":"2024-09-13T14:43:34.721642+0000","last_fullsized":"2024-09-13T14:43:34.721642+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":565,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.750159+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:43:34.750159+0000","last_peered":"2024-09-13T14:43:34.750159+0000","last_clean":"2024-09-13T14:43:34.750159+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:43:34.750159+0000","last_undegraded":"2024-09-13T14:43:34.750159+0000","last_fullsized":"2024-09-13T14:43:34.750159+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":569,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.731189+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:43:34.731189+0000","last_peered":"2024-09-13T14:43:34.731189+0000","last_clean":"2024-09-13T14:43:34.731189+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:43:34.731189+0000","last_undegraded":"2024-09-13T14:43:34.731189+0000","last_fullsized":"2024-09-13T14:43:34.731189+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":340,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.850231+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:43:25.850231+0000","last_peered":"2024-09-13T14:43:25.850231+0000","last_clean":"2024-09-13T14:43:25.850231+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:43:25.850231+0000","last_undegraded":"2024-09-13T14:43:25.850231+0000","last_fullsized":"2024-09-13T14:43:25.850231+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":587,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.731361+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:43:34.731361+0000","last_peered":"2024-09-13T14:43:34.731361+0000","last_clean":"2024-09-13T14:43:34.731361+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:43:34.731361+0000","last_undegraded":"2024-09-13T14:43:34.731361+0000","last_fullsized":"2024-09-13T14:43:34.731361+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":557,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.750160+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:43:34.750160+0000","last_peered":"2024-09-13T14:43:34.750160+0000","last_clean":"2024-09-13T14:43:34.750160+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:43:34.750160+0000","last_undegraded":"2024-09-13T14:43:34.750160+0000","last_fullsized":"2024-09-13T14:43:34.750160+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":533,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.729535+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:43:34.729535+0000","last_peered":"2024-09-13T14:43:34.729535+0000","last_clean":"2024-09-13T14:43:34.729535+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:43:34.729535+0000","last_undegraded":"2024-09-13T14:43:34.729535+0000","last_fullsized":"2024-09-13T14:43:34.729535+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":582,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.722382+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:43:34.722382+0000","last_peered":"2024-09-13T14:43:34.722382+0000","last_clean":"2024-09-13T14:43:34.722382+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:43:34.722382+0000","last_undegraded":"2024-09-13T14:43:34.722382+0000","last_fullsized":"2024-09-13T14:43:34.722382+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":41,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.744340+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:43:25.744340+0000","last_peered":"2024-09-13T14:43:25.744340+0000","last_clean":"2024-09-13T14:43:25.744340+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:43:25.744340+0000","last_undegraded":"2024-09-13T14:43:25.744340+0000","last_fullsized":"2024-09-13T14:43:25.744340+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":479,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.728969+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:43:34.728969+0000","last_peered":"2024-09-13T14:43:34.728969+0000","last_clean":"2024-09-13T14:43:34.728969+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:43:34.728969+0000","last_undegraded":"2024-09-13T14:43:34.728969+0000","last_fullsized":"2024-09-13T14:43:34.728969+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":560,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.749765+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:43:34.749765+0000","last_peered":"2024-09-13T14:43:34.749765+0000","last_clean":"2024-09-13T14:43:34.749765+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:43:34.749765+0000","last_undegraded":"2024-09-13T14:43:34.749765+0000","last_fullsized":"2024-09-13T14:43:34.749765+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.736407+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:43:25.736407+0000","last_peered":"2024-09-13T14:43:25.736407+0000","last_clean":"2024-09-13T14:43:25.736407+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:43:25.736407+0000","last_undegraded":"2024-09-13T14:43:25.736407+0000","last_fullsized":"2024-09-13T14:43:25.736407+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":308,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701298+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:43:25.701298+0000","last_peered":"2024-09-13T14:43:25.701298+0000","last_clean":"2024-09-13T14:43:25.701298+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:43:25.701298+0000","last_undegraded":"2024-09-13T14:43:25.701298+0000","last_fullsized":"2024-09-13T14:43:25.701298+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":41,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.744291+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:43:25.744291+0000","last_peered":"2024-09-13T14:43:25.744291+0000","last_clean":"2024-09-13T14:43:25.744291+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:43:25.744291+0000","last_undegraded":"2024-09-13T14:43:25.744291+0000","last_fullsized":"2024-09-13T14:43:25.744291+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":274,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.701792+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:43:25.701792+0000","last_peered":"2024-09-13T14:43:25.701792+0000","last_clean":"2024-09-13T14:43:25.701792+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:43:25.701792+0000","last_undegraded":"2024-09-13T14:43:25.701792+0000","last_fullsized":"2024-09-13T14:43:25.701792+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":579,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.722003+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:43:34.722003+0000","last_peered":"2024-09-13T14:43:34.722003+0000","last_clean":"2024-09-13T14:43:34.722003+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:43:34.722003+0000","last_undegraded":"2024-09-13T14:43:34.722003+0000","last_fullsized":"2024-09-13T14:43:34.722003+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":565,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.722802+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:43:34.722802+0000","last_peered":"2024-09-13T14:43:34.722802+0000","last_clean":"2024-09-13T14:43:34.722802+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:43:34.722802+0000","last_undegraded":"2024-09-13T14:43:34.722802+0000","last_fullsized":"2024-09-13T14:43:34.722802+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":42,"reported_epoch":342,"state":"active+clean","last_fresh":"2024-09-13T14:43:25.735704+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:43:25.735704+0000","last_peered":"2024-09-13T14:43:25.735704+0000","last_clean":"2024-09-13T14:43:25.735704+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:43:25.735704+0000","last_undegraded":"2024-09-13T14:43:25.735704+0000","last_fullsized":"2024-09-13T14:43:25.735704+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":570,"reported_epoch":343,"state":"active+clean","last_fresh":"2024-09-13T14:43:34.750056+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:43:34.750056+0000","last_peered":"2024-09-13T14:43:34.750056+0000","last_clean":"2024-09-13T14:43:34.750056+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:43:34.750056+0000","last_undegraded":"2024-09-13T14:43:34.750056+0000","last_fullsized":"2024-09-13T14:43:34.750056+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4699848728,"num_objects":71715,"num_object_clones":0,"num_object_copies":215145,"num_objects_missing_on_primary":3485,"num_objects_missing":3485,"num_objects_degraded":39363,"num_objects_misplaced":4436,"num_objects_unfound":0,"num_objects_dirty":71715,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110963,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22207,"num_bytes_recovered":1394933760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12892319744,"data_stored":12892307528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110963,"ondisk_log_size":110963,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397891,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2548396,"kb_used_data":1652388,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":91196756,"statfs":{"total":95995035648,"available":93385478144,"internally_reserved":0,"allocated":1692045312,"data_stored":1690260832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594119,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395672,"kb_used_data":1801616,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91349480,"statfs":{"total":95995035648,"available":93541867520,"internally_reserved":0,"allocated":1844854784,"data_stored":1843090058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665573,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670936,"kb_used_data":338384,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93074216,"statfs":{"total":95995035648,"available":95307997184,"internally_reserved":0,"allocated":346505216,"data_stored":344736821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986575,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1898916,"kb_used_data":1007388,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":91846236,"statfs":{"total":95995035648,"available":94050545664,"internally_reserved":0,"allocated":1031565312,"data_stored":1029788930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697135,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2803668,"kb_used_data":2089932,"kb_used_omap":4,"kb_used_meta":713723,"kb_avail":90941484,"statfs":{"total":95995035648,"available":93124079616,"internally_reserved":0,"allocated":2140090368,"data_stored":2138326126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":730852427},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860662,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2619612,"kb_used_data":2006676,"kb_used_omap":8,"kb_used_meta":612919,"kb_avail":91125540,"statfs":{"total":95995035648,"available":93312552960,"internally_reserved":0,"allocated":2054836224,"data_stored":2053064328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":627629900},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020333,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2310736,"kb_used_data":2011336,"kb_used_omap":9,"kb_used_meta":299382,"kb_avail":91434416,"statfs":{"total":95995035648,"available":93628841984,"internally_reserved":0,"allocated":2059608064,"data_stored":2057847295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":306568005},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187723,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2351568,"kb_used_data":1736904,"kb_used_omap":7,"kb_used_meta":614648,"kb_avail":91393584,"statfs":{"total":95995035648,"available":93587030016,"internally_reserved":0,"allocated":1778589696,"data_stored":1776829193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":629399883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1769340928,"data_stored":1769340928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2050883584,"data_stored":2050883584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2049380352,"data_stored":2049376280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2132803584,"data_stored":2132803584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1025966080,"data_stored":1025966080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":341250048,"data_stored":341245976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1839988736,"data_stored":1839988736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1682706432,"data_stored":1682702360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:50.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:50.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:50.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:50.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:43:50.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:50.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:50.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:50 smithi017 ceph-mon[103001]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39323/214533 objects degraded (18.330%); 4399/214533 objects misplaced (2.051%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:50.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:50 smithi017 ceph-mon[107556]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39323/214533 objects degraded (18.330%); 4399/214533 objects misplaced (2.051%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:50.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:50.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:50.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:50 smithi189 ceph-mon[90992]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 39323/214533 objects degraded (18.330%); 4399/214533 objects misplaced (2.051%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:43:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:43:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:50.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:50.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:50.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:50.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:50.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:50.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:50.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:50.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:50.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:51.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:51.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:51.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:51.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:51.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:43:51.291 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:51.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:51.314 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:43:51.314 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:51.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:51.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:43:51.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:51.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:51.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:43:51.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:51.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:43:51.364 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:51.379 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:43:51.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:51.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:43:51.396 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:51.411 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:43:51.411 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:51.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:51.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:51.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:51.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:51.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:51.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:51.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:51.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:51.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:51.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:51.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:52.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:52.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:52.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:52.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:52.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:52.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:52.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:52 smithi189 ceph-mon[90992]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 39323/213804 objects degraded (18.392%); 4399/213804 objects misplaced (2.057%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:52.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/806033041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:52.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:52.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:52.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:52.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[103001]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 39323/213804 objects degraded (18.392%); 4399/213804 objects misplaced (2.057%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:52.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/806033041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:52.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:52.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[107556]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 39323/213804 objects degraded (18.392%); 4399/213804 objects misplaced (2.057%); 1005 KiB/s, 15 objects/s recovering 2024-09-13T14:43:52.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/806033041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:52.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:52.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:52.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:52.819 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:52.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:52.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:52.857 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:52.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:52.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:52.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:53.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:53.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:53.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:53.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:43:53.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:53.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:53.241 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:53.241 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:53.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:53.344 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:53.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:53.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:53.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:53.596 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:53.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:53.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:53.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:53.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:53.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:53.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:54.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:54.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:54.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:54.221 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:54.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:43:54.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:54.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:54.486 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:54.510 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:54.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:54 smithi017 ceph-mon[103001]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 39283/213513 objects degraded (18.398%); 4399/213513 objects misplaced (2.060%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:43:54.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39283/213513 objects degraded (18.398%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:54.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:54 smithi017 ceph-mon[107556]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 39283/213513 objects degraded (18.398%); 4399/213513 objects misplaced (2.060%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:43:54.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39283/213513 objects degraded (18.398%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:54.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:54.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:54.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:54.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:54 smithi189 ceph-mon[90992]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 39283/213513 objects degraded (18.398%); 4399/213513 objects misplaced (2.060%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:43:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39283/213513 objects degraded (18.398%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:54.757 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:54.859 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:54.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:54.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:54.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:55.003 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:55.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:55.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:55.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:55.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:55.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:55.356 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:55.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:55.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:55.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:55.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:55.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:55.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:55.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:55.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:55.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:55.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:55.986 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:55.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:56.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:56.123 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:56.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:56.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:56.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:56 smithi017 ceph-mon[103001]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 39244/212814 objects degraded (18.441%); 4360/212814 objects misplaced (2.049%); 921 KiB/s, 14 objects/s recovering 2024-09-13T14:43:56.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:56 smithi017 ceph-mon[107556]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 39244/212814 objects degraded (18.441%); 4360/212814 objects misplaced (2.049%); 921 KiB/s, 14 objects/s recovering 2024-09-13T14:43:56.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:56.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:56.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:56 smithi189 ceph-mon[90992]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 39244/212814 objects degraded (18.441%); 4360/212814 objects misplaced (2.049%); 921 KiB/s, 14 objects/s recovering 2024-09-13T14:43:56.521 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:56.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:56.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:56.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:56.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:56.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:56.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:57.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:57.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:57.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:57.148 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:57.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:57.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:57.277 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:43:57.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:57.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:57.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:57.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:57.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:57.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:57.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:57.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:57.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:43:57] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:43:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:57.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:57.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:57.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:57.995 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:57.995 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:58.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.012 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:58.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:58.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:58.256 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:58.280 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:58.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:58.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 39192/212457 objects degraded (18.447%); 4360/212457 objects misplaced (2.052%); 781 KiB/s, 12 objects/s recovering 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 39192/212457 objects degraded (18.447%); 4360/212457 objects misplaced (2.052%); 781 KiB/s, 12 objects/s recovering 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:58.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:43:58.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:58.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:58.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:58.638 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 39192/212457 objects degraded (18.447%); 4360/212457 objects misplaced (2.052%); 781 KiB/s, 12 objects/s recovering 2024-09-13T14:43:58.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:58.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3874757682' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:58.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:43:58.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:43:58.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:58.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:43:58.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:58.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:58.971 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:43:59.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:59.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:59.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:59.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:59.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:59.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:59.380 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:43:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39192/212457 objects degraded (18.447%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:59.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:59.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:59.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:59.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:43:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39192/212457 objects degraded (18.447%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:43:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39192/212457 objects degraded (18.447%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:43:59.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:43:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:59.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:43:59.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:00.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:00.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:00.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:00.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:00.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:00.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:00 smithi189 ceph-mon[90992]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 39156/211827 objects degraded (18.485%); 4324/211827 objects misplaced (2.041%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:00.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:00.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:00.566 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:00 smithi017 ceph-mon[107556]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 39156/211827 objects degraded (18.485%); 4324/211827 objects misplaced (2.041%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:00.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:00 smithi017 ceph-mon[103001]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 90 op/s; 39156/211827 objects degraded (18.485%); 4324/211827 objects misplaced (2.041%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:00.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:00.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:00.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.851 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:00.851 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:00.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:44:00.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:00.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:01.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:01.050 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:01.050 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:01.056 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:01.056 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:01.058 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:01.074 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:01.074 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:01.095 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:01.095 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:01.112 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:01.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:01.131 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:01.131 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:01.149 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:01.149 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:01.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:01.168 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:01.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:01.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:01.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:01.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:01.419 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:01.445 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/626245774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/626245774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:01.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/626245774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:01.547 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:01.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:01.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:01.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:01.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:44:01.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:01.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:01.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:01.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:01.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:01.995 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:02.047 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:02.085 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:02.086 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:44:02.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:02.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:02.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:02.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:02.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:02.435 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:02 smithi189 ceph-mon[90992]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 39156/210942 objects degraded (18.562%); 4324/210942 objects misplaced (2.050%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:02.435 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:02 smithi189 ceph-mon[90992]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:02.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:02.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:02 smithi017 ceph-mon[103001]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 39156/210942 objects degraded (18.562%); 4324/210942 objects misplaced (2.050%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:02.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:02 smithi017 ceph-mon[103001]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:02.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:02 smithi017 ceph-mon[107556]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 39156/210942 objects degraded (18.562%); 4324/210942 objects misplaced (2.050%); 774 KiB/s, 12 objects/s recovering 2024-09-13T14:44:02.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:02 smithi017 ceph-mon[107556]: from='client.36273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:02.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:02.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:02.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:02.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:02.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:02.874 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1041,"stamp":"2024-09-13T14:44:01.199343+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4610886326,"num_objects":70314,"num_object_clones":0,"num_object_copies":210942,"num_objects_missing_on_primary":3370,"num_objects_missing":3370,"num_objects_degraded":39156,"num_objects_misplaced":4324,"num_objects_unfound":0,"num_objects_dirty":70314,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":113339,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22422,"num_bytes_recovered":1405454493,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114424,"ondisk_log_size":114424,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17373588,"kb_used_data":12402048,"kb_used_omap":44,"kb_used_meta":4971411,"kb_avail":732587628,"statfs":{"total":767960285184,"available":750169731072,"internally_reserved":0,"allocated":12699697152,"data_stored":12685428054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5090725111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-62521344,"num_objects":-954,"num_object_clones":0,"num_object_copies":-2862,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-167,"num_objects_misplaced":-75,"num_objects_unfound":0,"num_objects_dirty":-954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":954,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002770"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":573,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279755+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:43:57.279755+0000","last_peered":"2024-09-13T14:43:57.279755+0000","last_clean":"2024-09-13T14:43:57.279755+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:43:57.279755+0000","last_undegraded":"2024-09-13T14:43:57.279755+0000","last_fullsized":"2024-09-13T14:43:57.279755+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":343,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.373256+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:43:47.373256+0000","last_peered":"2024-09-13T14:43:47.373256+0000","last_clean":"2024-09-13T14:43:47.373256+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:43:47.373256+0000","last_undegraded":"2024-09-13T14:43:47.373256+0000","last_fullsized":"2024-09-13T14:43:47.373256+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":272,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264352+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:43:47.264352+0000","last_peered":"2024-09-13T14:43:47.264352+0000","last_clean":"2024-09-13T14:43:47.264352+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:43:47.264352+0000","last_undegraded":"2024-09-13T14:43:47.264352+0000","last_fullsized":"2024-09-13T14:43:47.264352+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":558,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378811+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:43:57.378811+0000","last_peered":"2024-09-13T14:43:57.378811+0000","last_clean":"2024-09-13T14:43:57.378811+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:43:57.378811+0000","last_undegraded":"2024-09-13T14:43:57.378811+0000","last_fullsized":"2024-09-13T14:43:57.378811+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":559,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378912+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:43:57.378912+0000","last_peered":"2024-09-13T14:43:57.378912+0000","last_clean":"2024-09-13T14:43:57.378912+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:43:57.378912+0000","last_undegraded":"2024-09-13T14:43:57.378912+0000","last_fullsized":"2024-09-13T14:43:57.378912+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":312,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.373136+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:43:47.373136+0000","last_peered":"2024-09-13T14:43:47.373136+0000","last_clean":"2024-09-13T14:43:47.373136+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:43:47.373136+0000","last_undegraded":"2024-09-13T14:43:47.373136+0000","last_fullsized":"2024-09-13T14:43:47.373136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":573,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280071+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:43:57.280071+0000","last_peered":"2024-09-13T14:43:57.280071+0000","last_clean":"2024-09-13T14:43:57.280071+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:43:57.280071+0000","last_undegraded":"2024-09-13T14:43:57.280071+0000","last_fullsized":"2024-09-13T14:43:57.280071+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":334,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354501+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:43:47.354501+0000","last_peered":"2024-09-13T14:43:47.354501+0000","last_clean":"2024-09-13T14:43:47.354501+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:43:47.354501+0000","last_undegraded":"2024-09-13T14:43:47.354501+0000","last_fullsized":"2024-09-13T14:43:47.354501+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"344'6934","reported_seq":10889,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:56.591286+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:43:56.591286+0000","last_peered":"2024-09-13T14:43:56.591286+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:43:56.591286+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6934,"log_dups_size":0,"ondisk_log_size":6934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285474816,"num_objects":4356,"num_object_clones":0,"num_object_copies":13068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6934,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1289,"num_bytes_recovered":84475904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3372}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":338,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.373663+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:43:47.373663+0000","last_peered":"2024-09-13T14:43:47.373663+0000","last_clean":"2024-09-13T14:43:47.373663+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:43:47.373663+0000","last_undegraded":"2024-09-13T14:43:47.373663+0000","last_fullsized":"2024-09-13T14:43:47.373663+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.263966+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:43:47.263966+0000","last_peered":"2024-09-13T14:43:47.263966+0000","last_clean":"2024-09-13T14:43:47.263966+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:43:47.263966+0000","last_undegraded":"2024-09-13T14:43:47.263966+0000","last_fullsized":"2024-09-13T14:43:47.263966+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"345'7048","reported_seq":16590,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.758234+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:43:57.758234+0000","last_peered":"2024-09-13T14:43:57.758234+0000","last_clean":"2024-09-13T14:43:57.758234+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:43:57.758234+0000","last_undegraded":"2024-09-13T14:43:57.758234+0000","last_fullsized":"2024-09-13T14:43:57.758234+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290979840,"num_objects":4440,"num_object_clones":0,"num_object_copies":13320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7048,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":356,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.373626+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:43:47.373626+0000","last_peered":"2024-09-13T14:43:47.373626+0000","last_clean":"2024-09-13T14:43:47.373626+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:43:47.373626+0000","last_undegraded":"2024-09-13T14:43:47.373626+0000","last_fullsized":"2024-09-13T14:43:47.373626+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303277+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:43:47.303277+0000","last_peered":"2024-09-13T14:43:47.303277+0000","last_clean":"2024-09-13T14:43:47.303277+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:43:47.303277+0000","last_undegraded":"2024-09-13T14:43:47.303277+0000","last_fullsized":"2024-09-13T14:43:47.303277+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"345'7239","reported_seq":7336,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:58.728103+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:43:58.728103+0000","last_peered":"2024-09-13T14:43:58.728103+0000","last_clean":"2024-09-13T14:43:58.728103+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:43:58.728103+0000","last_undegraded":"2024-09-13T14:43:58.728103+0000","last_fullsized":"2024-09-13T14:43:58.728103+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7239,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":310,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.375560+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:43:47.375560+0000","last_peered":"2024-09-13T14:43:47.375560+0000","last_clean":"2024-09-13T14:43:47.375560+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:43:47.375560+0000","last_undegraded":"2024-09-13T14:43:47.375560+0000","last_fullsized":"2024-09-13T14:43:47.375560+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":325,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354607+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:43:47.354607+0000","last_peered":"2024-09-13T14:43:47.354607+0000","last_clean":"2024-09-13T14:43:47.354607+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:43:47.354607+0000","last_undegraded":"2024-09-13T14:43:47.354607+0000","last_fullsized":"2024-09-13T14:43:47.354607+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"345'7127","reported_seq":7224,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:59.615428+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:43:59.615428+0000","last_peered":"2024-09-13T14:43:59.615428+0000","last_clean":"2024-09-13T14:43:59.615428+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:43:59.615428+0000","last_undegraded":"2024-09-13T14:43:59.615428+0000","last_fullsized":"2024-09-13T14:43:59.615428+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7127,"log_dups_size":0,"ondisk_log_size":7127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289603584,"num_objects":4419,"num_object_clones":0,"num_object_copies":13257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7127,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":273,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264423+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:43:47.264423+0000","last_peered":"2024-09-13T14:43:47.264423+0000","last_clean":"2024-09-13T14:43:47.264423+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:43:47.264423+0000","last_undegraded":"2024-09-13T14:43:47.264423+0000","last_fullsized":"2024-09-13T14:43:47.264423+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":584,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378451+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:43:57.378451+0000","last_peered":"2024-09-13T14:43:57.378451+0000","last_clean":"2024-09-13T14:43:57.378451+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:43:57.378451+0000","last_undegraded":"2024-09-13T14:43:57.378451+0000","last_fullsized":"2024-09-13T14:43:57.378451+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"345'7068","reported_seq":7162,"reported_epoch":345,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:00.222168+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:44:00.222168+0000","last_peered":"2024-09-13T14:44:00.222168+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:44:00.222168+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287965184,"num_objects":4394,"num_object_clones":0,"num_object_copies":13182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7068,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1239}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":456,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.372913+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:43:47.372913+0000","last_peered":"2024-09-13T14:43:47.372913+0000","last_clean":"2024-09-13T14:43:47.372913+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:43:47.372913+0000","last_undegraded":"2024-09-13T14:43:47.372913+0000","last_fullsized":"2024-09-13T14:43:47.372913+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":582,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279885+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:43:57.279885+0000","last_peered":"2024-09-13T14:43:57.279885+0000","last_clean":"2024-09-13T14:43:57.279885+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:43:57.279885+0000","last_undegraded":"2024-09-13T14:43:57.279885+0000","last_fullsized":"2024-09-13T14:43:57.279885+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303524+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:43:47.303524+0000","last_peered":"2024-09-13T14:43:47.303524+0000","last_clean":"2024-09-13T14:43:47.303524+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:43:47.303524+0000","last_undegraded":"2024-09-13T14:43:47.303524+0000","last_fullsized":"2024-09-13T14:43:47.303524+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"345'6807","reported_seq":6904,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:59.720809+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:43:59.720809+0000","last_peered":"2024-09-13T14:43:59.720809+0000","last_clean":"2024-09-13T14:43:59.720809+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:43:59.720809+0000","last_undegraded":"2024-09-13T14:43:59.720809+0000","last_fullsized":"2024-09-13T14:43:59.720809+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6807,"log_dups_size":0,"ondisk_log_size":6807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280952832,"num_objects":4287,"num_object_clones":0,"num_object_copies":12861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6807,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":160,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.263975+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:43:47.263975+0000","last_peered":"2024-09-13T14:43:47.263975+0000","last_clean":"2024-09-13T14:43:47.263975+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:43:47.263975+0000","last_undegraded":"2024-09-13T14:43:47.263975+0000","last_fullsized":"2024-09-13T14:43:47.263975+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":346,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354540+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:43:47.354540+0000","last_peered":"2024-09-13T14:43:47.354540+0000","last_clean":"2024-09-13T14:43:47.354540+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:43:47.354540+0000","last_undegraded":"2024-09-13T14:43:47.354540+0000","last_fullsized":"2024-09-13T14:43:47.354540+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303621+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:43:47.303621+0000","last_peered":"2024-09-13T14:43:47.303621+0000","last_clean":"2024-09-13T14:43:47.303621+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:43:47.303621+0000","last_undegraded":"2024-09-13T14:43:47.303621+0000","last_fullsized":"2024-09-13T14:43:47.303621+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"345'6985","reported_seq":10530,"reported_epoch":345,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:58.794324+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:43:58.794324+0000","last_peered":"2024-09-13T14:43:58.794324+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:43:58.794324+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290127872,"num_objects":4427,"num_object_clones":0,"num_object_copies":13281,"num_objects_missing_on_primary":3370,"num_objects_missing":3370,"num_objects_degraded":3745,"num_objects_misplaced":4324,"num_objects_unfound":0,"num_objects_dirty":4427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6985,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1279,"num_bytes_recovered":83820544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3370},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303123+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:43:47.303123+0000","last_peered":"2024-09-13T14:43:47.303123+0000","last_clean":"2024-09-13T14:43:47.303123+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:43:47.303123+0000","last_undegraded":"2024-09-13T14:43:47.303123+0000","last_fullsized":"2024-09-13T14:43:47.303123+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303684+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:43:47.303684+0000","last_peered":"2024-09-13T14:43:47.303684+0000","last_clean":"2024-09-13T14:43:47.303684+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:43:47.303684+0000","last_undegraded":"2024-09-13T14:43:47.303684+0000","last_fullsized":"2024-09-13T14:43:47.303684+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"345'6859","reported_seq":6956,"reported_epoch":345,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:59.713584+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:43:59.713584+0000","last_peered":"2024-09-13T14:43:59.713584+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:43:59.713584+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281739264,"num_objects":4299,"num_object_clones":0,"num_object_copies":12897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6859,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1167}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":153,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264160+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:43:47.264160+0000","last_peered":"2024-09-13T14:43:47.264160+0000","last_clean":"2024-09-13T14:43:47.264160+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:43:47.264160+0000","last_undegraded":"2024-09-13T14:43:47.264160+0000","last_fullsized":"2024-09-13T14:43:47.264160+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":575,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.286464+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:43:57.286464+0000","last_peered":"2024-09-13T14:43:57.286464+0000","last_clean":"2024-09-13T14:43:57.286464+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:43:57.286464+0000","last_undegraded":"2024-09-13T14:43:57.286464+0000","last_fullsized":"2024-09-13T14:43:57.286464+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":537,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295957+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:43:57.295957+0000","last_peered":"2024-09-13T14:43:57.295957+0000","last_clean":"2024-09-13T14:43:57.295957+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:43:57.295957+0000","last_undegraded":"2024-09-13T14:43:57.295957+0000","last_fullsized":"2024-09-13T14:43:57.295957+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"344'6884","reported_seq":6941,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:57.101391+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:43:57.101391+0000","last_peered":"2024-09-13T14:43:57.101391+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:43:57.101391+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284164096,"num_objects":4336,"num_object_clones":0,"num_object_copies":13008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6884,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1160}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":447,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.372992+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:43:47.372992+0000","last_peered":"2024-09-13T14:43:47.372992+0000","last_clean":"2024-09-13T14:43:47.372992+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:43:47.372992+0000","last_undegraded":"2024-09-13T14:43:47.372992+0000","last_fullsized":"2024-09-13T14:43:47.372992+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":869,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264354+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:43:47.264354+0000","last_peered":"2024-09-13T14:43:47.264354+0000","last_clean":"2024-09-13T14:43:47.264354+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:43:47.264354+0000","last_undegraded":"2024-09-13T14:43:47.264354+0000","last_fullsized":"2024-09-13T14:43:47.264354+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":528,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279277+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:43:57.279277+0000","last_peered":"2024-09-13T14:43:57.279277+0000","last_clean":"2024-09-13T14:43:57.279277+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:43:57.279277+0000","last_undegraded":"2024-09-13T14:43:57.279277+0000","last_fullsized":"2024-09-13T14:43:57.279277+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":311,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.355135+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:43:47.355135+0000","last_peered":"2024-09-13T14:43:47.355135+0000","last_clean":"2024-09-13T14:43:47.355135+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:43:47.355135+0000","last_undegraded":"2024-09-13T14:43:47.355135+0000","last_fullsized":"2024-09-13T14:43:47.355135+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"345'7130","reported_seq":7225,"reported_epoch":345,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:43:59.559055+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:43:59.559055+0000","last_peered":"2024-09-13T14:43:59.559055+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:43:59.559055+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7130,"log_dups_size":0,"ondisk_log_size":7130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289538048,"num_objects":4418,"num_object_clones":0,"num_object_copies":13254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7130,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1262}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":382,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295229+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:43:57.295229+0000","last_peered":"2024-09-13T14:43:57.295229+0000","last_clean":"2024-09-13T14:43:57.295229+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:43:57.295229+0000","last_undegraded":"2024-09-13T14:43:57.295229+0000","last_fullsized":"2024-09-13T14:43:57.295229+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":735,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.372967+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:43:47.372967+0000","last_peered":"2024-09-13T14:43:47.372967+0000","last_clean":"2024-09-13T14:43:47.372967+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:43:47.372967+0000","last_undegraded":"2024-09-13T14:43:47.372967+0000","last_fullsized":"2024-09-13T14:43:47.372967+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":577,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378634+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:43:57.378634+0000","last_peered":"2024-09-13T14:43:57.378634+0000","last_clean":"2024-09-13T14:43:57.378634+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:43:57.378634+0000","last_undegraded":"2024-09-13T14:43:57.378634+0000","last_fullsized":"2024-09-13T14:43:57.378634+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":582,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378831+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:43:57.378831+0000","last_peered":"2024-09-13T14:43:57.378831+0000","last_clean":"2024-09-13T14:43:57.378831+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:43:57.378831+0000","last_undegraded":"2024-09-13T14:43:57.378831+0000","last_fullsized":"2024-09-13T14:43:57.378831+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"345'7148","reported_seq":16785,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:44:00.402128+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:44:00.402128+0000","last_peered":"2024-09-13T14:44:00.402128+0000","last_clean":"2024-09-13T14:44:00.402128+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:44:00.402128+0000","last_undegraded":"2024-09-13T14:44:00.402128+0000","last_fullsized":"2024-09-13T14:44:00.402128+0000","mapping_epoch":330,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7148,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":476,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.373016+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:43:47.373016+0000","last_peered":"2024-09-13T14:43:47.373016+0000","last_clean":"2024-09-13T14:43:47.373016+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:43:47.373016+0000","last_undegraded":"2024-09-13T14:43:47.373016+0000","last_fullsized":"2024-09-13T14:43:47.373016+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":546,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279511+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:43:57.279511+0000","last_peered":"2024-09-13T14:43:57.279511+0000","last_clean":"2024-09-13T14:43:57.279511+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:43:57.279511+0000","last_undegraded":"2024-09-13T14:43:57.279511+0000","last_fullsized":"2024-09-13T14:43:57.279511+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":514,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.294926+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:43:57.294926+0000","last_peered":"2024-09-13T14:43:57.294926+0000","last_clean":"2024-09-13T14:43:57.294926+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:43:57.294926+0000","last_undegraded":"2024-09-13T14:43:57.294926+0000","last_fullsized":"2024-09-13T14:43:57.294926+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264077+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:43:47.264077+0000","last_peered":"2024-09-13T14:43:47.264077+0000","last_clean":"2024-09-13T14:43:47.264077+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:43:47.264077+0000","last_undegraded":"2024-09-13T14:43:47.264077+0000","last_fullsized":"2024-09-13T14:43:47.264077+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"345'6887","reported_seq":6983,"reported_epoch":345,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:00.438184+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:44:00.438184+0000","last_peered":"2024-09-13T14:44:00.438184+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:44:00.438184+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284622848,"num_objects":4343,"num_object_clones":0,"num_object_copies":13029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6887,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1166}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":153,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264000+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:43:47.264000+0000","last_peered":"2024-09-13T14:43:47.264000+0000","last_clean":"2024-09-13T14:43:47.264000+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:43:47.264000+0000","last_undegraded":"2024-09-13T14:43:47.264000+0000","last_fullsized":"2024-09-13T14:43:47.264000+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":960,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264185+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:43:47.264185+0000","last_peered":"2024-09-13T14:43:47.264185+0000","last_clean":"2024-09-13T14:43:47.264185+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:43:47.264185+0000","last_undegraded":"2024-09-13T14:43:47.264185+0000","last_fullsized":"2024-09-13T14:43:47.264185+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303413+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:43:47.303413+0000","last_peered":"2024-09-13T14:43:47.303413+0000","last_clean":"2024-09-13T14:43:47.303413+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:43:47.303413+0000","last_undegraded":"2024-09-13T14:43:47.303413+0000","last_fullsized":"2024-09-13T14:43:47.303413+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":535,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.294948+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:43:57.294948+0000","last_peered":"2024-09-13T14:43:57.294948+0000","last_clean":"2024-09-13T14:43:57.294948+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:43:57.294948+0000","last_undegraded":"2024-09-13T14:43:57.294948+0000","last_fullsized":"2024-09-13T14:43:57.294948+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"345'6987","reported_seq":16363,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:58.731534+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:43:58.731534+0000","last_peered":"2024-09-13T14:43:58.731534+0000","last_clean":"2024-09-13T14:43:58.731534+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:43:58.731534+0000","last_undegraded":"2024-09-13T14:43:58.731534+0000","last_fullsized":"2024-09-13T14:43:58.731534+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6987,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":344,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354651+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:43:47.354651+0000","last_peered":"2024-09-13T14:43:47.354651+0000","last_clean":"2024-09-13T14:43:47.354651+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:43:47.354651+0000","last_undegraded":"2024-09-13T14:43:47.354651+0000","last_fullsized":"2024-09-13T14:43:47.354651+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":290,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264217+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:43:47.264217+0000","last_peered":"2024-09-13T14:43:47.264217+0000","last_clean":"2024-09-13T14:43:47.264217+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:43:47.264217+0000","last_undegraded":"2024-09-13T14:43:47.264217+0000","last_fullsized":"2024-09-13T14:43:47.264217+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1114,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295222+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:43:57.295222+0000","last_peered":"2024-09-13T14:43:57.295222+0000","last_clean":"2024-09-13T14:43:57.295222+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:43:57.295222+0000","last_undegraded":"2024-09-13T14:43:57.295222+0000","last_fullsized":"2024-09-13T14:43:57.295222+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":516,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295553+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:43:57.295553+0000","last_peered":"2024-09-13T14:43:57.295553+0000","last_clean":"2024-09-13T14:43:57.295553+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:43:57.295553+0000","last_undegraded":"2024-09-13T14:43:57.295553+0000","last_fullsized":"2024-09-13T14:43:57.295553+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"345'6926","reported_seq":7022,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:44:00.589770+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:44:00.589770+0000","last_peered":"2024-09-13T14:44:00.589770+0000","last_clean":"2024-09-13T14:44:00.589770+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:44:00.589770+0000","last_undegraded":"2024-09-13T14:44:00.589770+0000","last_fullsized":"2024-09-13T14:44:00.589770+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6926,"log_dups_size":0,"ondisk_log_size":6926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280363008,"num_objects":4278,"num_object_clones":0,"num_object_copies":12834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6926,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":344,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354744+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:43:47.354744+0000","last_peered":"2024-09-13T14:43:47.354744+0000","last_clean":"2024-09-13T14:43:47.354744+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:43:47.354744+0000","last_undegraded":"2024-09-13T14:43:47.354744+0000","last_fullsized":"2024-09-13T14:43:47.354744+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":525,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295091+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:43:57.295091+0000","last_peered":"2024-09-13T14:43:57.295091+0000","last_clean":"2024-09-13T14:43:57.295091+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:43:57.295091+0000","last_undegraded":"2024-09-13T14:43:57.295091+0000","last_fullsized":"2024-09-13T14:43:57.295091+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":739,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.375612+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:43:47.375612+0000","last_peered":"2024-09-13T14:43:47.375612+0000","last_clean":"2024-09-13T14:43:47.375612+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:43:47.375612+0000","last_undegraded":"2024-09-13T14:43:47.375612+0000","last_fullsized":"2024-09-13T14:43:47.375612+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":560,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378350+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:43:57.378350+0000","last_peered":"2024-09-13T14:43:57.378350+0000","last_clean":"2024-09-13T14:43:57.378350+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:43:57.378350+0000","last_undegraded":"2024-09-13T14:43:57.378350+0000","last_fullsized":"2024-09-13T14:43:57.378350+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"345'7243","reported_seq":7339,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:58.664769+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:43:58.664769+0000","last_peered":"2024-09-13T14:43:58.664769+0000","last_clean":"2024-09-13T14:43:58.664769+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:43:58.664769+0000","last_undegraded":"2024-09-13T14:43:58.664769+0000","last_fullsized":"2024-09-13T14:43:58.664769+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7243,"log_dups_size":0,"ondisk_log_size":7243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291438592,"num_objects":4447,"num_object_clones":0,"num_object_copies":13341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7243,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303432+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:43:47.303432+0000","last_peered":"2024-09-13T14:43:47.303432+0000","last_clean":"2024-09-13T14:43:47.303432+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:43:47.303432+0000","last_undegraded":"2024-09-13T14:43:47.303432+0000","last_fullsized":"2024-09-13T14:43:47.303432+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":561,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280349+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:43:57.280349+0000","last_peered":"2024-09-13T14:43:57.280349+0000","last_clean":"2024-09-13T14:43:57.280349+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:43:57.280349+0000","last_undegraded":"2024-09-13T14:43:57.280349+0000","last_fullsized":"2024-09-13T14:43:57.280349+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":901,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295699+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:43:57.295699+0000","last_peered":"2024-09-13T14:43:57.295699+0000","last_clean":"2024-09-13T14:43:57.295699+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:43:57.295699+0000","last_undegraded":"2024-09-13T14:43:57.295699+0000","last_fullsized":"2024-09-13T14:43:57.295699+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":559,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.286378+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:43:57.286378+0000","last_peered":"2024-09-13T14:43:57.286378+0000","last_clean":"2024-09-13T14:43:57.286378+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:43:57.286378+0000","last_undegraded":"2024-09-13T14:43:57.286378+0000","last_fullsized":"2024-09-13T14:43:57.286378+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"345'7120","reported_seq":18476,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.623227+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:43:57.623227+0000","last_peered":"2024-09-13T14:43:57.623227+0000","last_clean":"2024-09-13T14:43:57.623227+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:43:57.623227+0000","last_undegraded":"2024-09-13T14:43:57.623227+0000","last_fullsized":"2024-09-13T14:43:57.623227+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7120,"log_dups_size":0,"ondisk_log_size":7120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899672,"num_objects":4394,"num_object_clones":0,"num_object_copies":13182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7120,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":412,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279584+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:43:57.279584+0000","last_peered":"2024-09-13T14:43:57.279584+0000","last_clean":"2024-09-13T14:43:57.279584+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:43:57.279584+0000","last_undegraded":"2024-09-13T14:43:57.279584+0000","last_fullsized":"2024-09-13T14:43:57.279584+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":596,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378609+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:43:57.378609+0000","last_peered":"2024-09-13T14:43:57.378609+0000","last_clean":"2024-09-13T14:43:57.378609+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:43:57.378609+0000","last_undegraded":"2024-09-13T14:43:57.378609+0000","last_fullsized":"2024-09-13T14:43:57.378609+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":763,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354789+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:43:47.354789+0000","last_peered":"2024-09-13T14:43:47.354789+0000","last_clean":"2024-09-13T14:43:47.354789+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:43:47.354789+0000","last_undegraded":"2024-09-13T14:43:47.354789+0000","last_fullsized":"2024-09-13T14:43:47.354789+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303206+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:43:47.303206+0000","last_peered":"2024-09-13T14:43:47.303206+0000","last_clean":"2024-09-13T14:43:47.303206+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:43:47.303206+0000","last_undegraded":"2024-09-13T14:43:47.303206+0000","last_fullsized":"2024-09-13T14:43:47.303206+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303431+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:43:47.303431+0000","last_peered":"2024-09-13T14:43:47.303431+0000","last_clean":"2024-09-13T14:43:47.303431+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:43:47.303431+0000","last_undegraded":"2024-09-13T14:43:47.303431+0000","last_fullsized":"2024-09-13T14:43:47.303431+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":54,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303381+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:43:47.303381+0000","last_peered":"2024-09-13T14:43:47.303381+0000","last_clean":"2024-09-13T14:43:47.303381+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:43:47.303381+0000","last_undegraded":"2024-09-13T14:43:47.303381+0000","last_fullsized":"2024-09-13T14:43:47.303381+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":512,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295413+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:43:57.295413+0000","last_peered":"2024-09-13T14:43:57.295413+0000","last_clean":"2024-09-13T14:43:57.295413+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:43:57.295413+0000","last_undegraded":"2024-09-13T14:43:57.295413+0000","last_fullsized":"2024-09-13T14:43:57.295413+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":344,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.375505+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:43:47.375505+0000","last_peered":"2024-09-13T14:43:47.375505+0000","last_clean":"2024-09-13T14:43:47.375505+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:43:47.375505+0000","last_undegraded":"2024-09-13T14:43:47.375505+0000","last_fullsized":"2024-09-13T14:43:47.375505+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":310,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264103+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:43:47.264103+0000","last_peered":"2024-09-13T14:43:47.264103+0000","last_clean":"2024-09-13T14:43:47.264103+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:43:47.264103+0000","last_undegraded":"2024-09-13T14:43:47.264103+0000","last_fullsized":"2024-09-13T14:43:47.264103+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":517,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279242+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:43:57.279242+0000","last_peered":"2024-09-13T14:43:57.279242+0000","last_clean":"2024-09-13T14:43:57.279242+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:43:57.279242+0000","last_undegraded":"2024-09-13T14:43:57.279242+0000","last_fullsized":"2024-09-13T14:43:57.279242+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":569,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378527+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:43:57.378527+0000","last_peered":"2024-09-13T14:43:57.378527+0000","last_clean":"2024-09-13T14:43:57.378527+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:43:57.378527+0000","last_undegraded":"2024-09-13T14:43:57.378527+0000","last_fullsized":"2024-09-13T14:43:57.378527+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":573,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.286309+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:43:57.286309+0000","last_peered":"2024-09-13T14:43:57.286309+0000","last_clean":"2024-09-13T14:43:57.286309+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:43:57.286309+0000","last_undegraded":"2024-09-13T14:43:57.286309+0000","last_fullsized":"2024-09-13T14:43:57.286309+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":342,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.354717+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:43:47.354717+0000","last_peered":"2024-09-13T14:43:47.354717+0000","last_clean":"2024-09-13T14:43:47.354717+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:43:47.354717+0000","last_undegraded":"2024-09-13T14:43:47.354717+0000","last_fullsized":"2024-09-13T14:43:47.354717+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":591,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.286406+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:43:57.286406+0000","last_peered":"2024-09-13T14:43:57.286406+0000","last_clean":"2024-09-13T14:43:57.286406+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:43:57.286406+0000","last_undegraded":"2024-09-13T14:43:57.286406+0000","last_fullsized":"2024-09-13T14:43:57.286406+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":561,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378748+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:43:57.378748+0000","last_peered":"2024-09-13T14:43:57.378748+0000","last_clean":"2024-09-13T14:43:57.378748+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:43:57.378748+0000","last_undegraded":"2024-09-13T14:43:57.378748+0000","last_fullsized":"2024-09-13T14:43:57.378748+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":537,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.294875+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:43:57.294875+0000","last_peered":"2024-09-13T14:43:57.294875+0000","last_clean":"2024-09-13T14:43:57.294875+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:43:57.294875+0000","last_undegraded":"2024-09-13T14:43:57.294875+0000","last_fullsized":"2024-09-13T14:43:57.294875+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":587,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279437+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:43:57.279437+0000","last_peered":"2024-09-13T14:43:57.279437+0000","last_clean":"2024-09-13T14:43:57.279437+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:43:57.279437+0000","last_undegraded":"2024-09-13T14:43:57.279437+0000","last_fullsized":"2024-09-13T14:43:57.279437+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":43,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.315992+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:43:47.315992+0000","last_peered":"2024-09-13T14:43:47.315992+0000","last_clean":"2024-09-13T14:43:47.315992+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:43:47.315992+0000","last_undegraded":"2024-09-13T14:43:47.315992+0000","last_fullsized":"2024-09-13T14:43:47.315992+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":483,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.295362+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:43:57.295362+0000","last_peered":"2024-09-13T14:43:57.295362+0000","last_clean":"2024-09-13T14:43:57.295362+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:43:57.295362+0000","last_undegraded":"2024-09-13T14:43:57.295362+0000","last_fullsized":"2024-09-13T14:43:57.295362+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":564,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378350+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:43:57.378350+0000","last_peered":"2024-09-13T14:43:57.378350+0000","last_clean":"2024-09-13T14:43:57.378350+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:43:57.378350+0000","last_undegraded":"2024-09-13T14:43:57.378350+0000","last_fullsized":"2024-09-13T14:43:57.378350+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303361+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:43:47.303361+0000","last_peered":"2024-09-13T14:43:47.303361+0000","last_clean":"2024-09-13T14:43:47.303361+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:43:47.303361+0000","last_undegraded":"2024-09-13T14:43:47.303361+0000","last_fullsized":"2024-09-13T14:43:47.303361+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":310,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264411+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:43:47.264411+0000","last_peered":"2024-09-13T14:43:47.264411+0000","last_clean":"2024-09-13T14:43:47.264411+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:43:47.264411+0000","last_undegraded":"2024-09-13T14:43:47.264411+0000","last_fullsized":"2024-09-13T14:43:47.264411+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.315896+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:43:47.315896+0000","last_peered":"2024-09-13T14:43:47.315896+0000","last_clean":"2024-09-13T14:43:47.315896+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:43:47.315896+0000","last_undegraded":"2024-09-13T14:43:47.315896+0000","last_fullsized":"2024-09-13T14:43:47.315896+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":276,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.264397+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:43:47.264397+0000","last_peered":"2024-09-13T14:43:47.264397+0000","last_clean":"2024-09-13T14:43:47.264397+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:43:47.264397+0000","last_undegraded":"2024-09-13T14:43:47.264397+0000","last_fullsized":"2024-09-13T14:43:47.264397+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":583,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279606+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:43:57.279606+0000","last_peered":"2024-09-13T14:43:57.279606+0000","last_clean":"2024-09-13T14:43:57.279606+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:43:57.279606+0000","last_undegraded":"2024-09-13T14:43:57.279606+0000","last_fullsized":"2024-09-13T14:43:57.279606+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":570,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279865+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:43:57.279865+0000","last_peered":"2024-09-13T14:43:57.279865+0000","last_clean":"2024-09-13T14:43:57.279865+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:43:57.279865+0000","last_undegraded":"2024-09-13T14:43:57.279865+0000","last_fullsized":"2024-09-13T14:43:57.279865+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":44,"reported_epoch":344,"state":"active+clean","last_fresh":"2024-09-13T14:43:47.303095+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:43:47.303095+0000","last_peered":"2024-09-13T14:43:47.303095+0000","last_clean":"2024-09-13T14:43:47.303095+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:43:47.303095+0000","last_undegraded":"2024-09-13T14:43:47.303095+0000","last_fullsized":"2024-09-13T14:43:47.303095+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":574,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.378394+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:43:57.378394+0000","last_peered":"2024-09-13T14:43:57.378394+0000","last_clean":"2024-09-13T14:43:57.378394+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:43:57.378394+0000","last_undegraded":"2024-09-13T14:43:57.378394+0000","last_fullsized":"2024-09-13T14:43:57.378394+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4606197784,"num_objects":70286,"num_object_clones":0,"num_object_copies":210858,"num_objects_missing_on_primary":3370,"num_objects_missing":3370,"num_objects_degraded":39156,"num_objects_misplaced":4324,"num_objects_unfound":0,"num_objects_dirty":70286,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112392,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22365,"num_bytes_recovered":1405288448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12644462592,"data_stored":12644450376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112292,"ondisk_log_size":112292,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397894,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2495888,"kb_used_data":1595716,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91249264,"statfs":{"total":95995035648,"available":93439246336,"internally_reserved":0,"allocated":1634013184,"data_stored":1632213010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594122,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367804,"kb_used_data":1773744,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91377348,"statfs":{"total":95995035648,"available":93570404352,"internally_reserved":0,"allocated":1816313856,"data_stored":1814533436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665576,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":669756,"kb_used_data":333040,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93075396,"statfs":{"total":95995035648,"available":95309205504,"internally_reserved":0,"allocated":341032960,"data_stored":339248871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986578,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883528,"kb_used_data":987836,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":91861624,"statfs":{"total":95995035648,"available":94066302976,"internally_reserved":0,"allocated":1011544064,"data_stored":1009751988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697138,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2767988,"kb_used_data":2054252,"kb_used_omap":4,"kb_used_meta":713723,"kb_avail":90977164,"statfs":{"total":95995035648,"available":93160615936,"internally_reserved":0,"allocated":2103554048,"data_stored":2101774112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":730852427},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860664,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2600364,"kb_used_data":1987428,"kb_used_omap":8,"kb_used_meta":612919,"kb_avail":91144788,"statfs":{"total":95995035648,"available":93332262912,"internally_reserved":0,"allocated":2035126272,"data_stored":2033346529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":627629900},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020336,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269168,"kb_used_data":1969768,"kb_used_omap":9,"kb_used_meta":299382,"kb_avail":91475984,"statfs":{"total":95995035648,"available":93671407616,"internally_reserved":0,"allocated":2017042432,"data_stored":2015265969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":306568005},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187726,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2319092,"kb_used_data":1700264,"kb_used_omap":7,"kb_used_meta":618808,"kb_avail":91426060,"statfs":{"total":95995035648,"available":93620285440,"internally_reserved":0,"allocated":1741070336,"data_stored":1739294139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":633659723},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1731919872,"data_stored":1731919872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2007826432,"data_stored":2007826432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2030309376,"data_stored":2030305304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2095448064,"data_stored":2095448064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1007157248,"data_stored":1007157248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":335745024,"data_stored":335740952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1811480576,"data_stored":1811480576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1624576000,"data_stored":1624571928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:02.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:44:02.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:02.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:02.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:03.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:03.178 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:03.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:03.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:03.314 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:03.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:03.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:03.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.691 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:03.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:03.813 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89683 8.8 8.e68ed018 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 200.08378274500001, 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98019932300000001, 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:44:03.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89683 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719757+0000", 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719930+0000", 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017338299999999999 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719942+0000", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1644999999999999e-05 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719950+0000", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.622e-06 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.720009+0000", 2024-09-13T14:44:03.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9120000000000003e-05 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.720018+0000", 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7010000000000002e-06 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.699946+0000", 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97992809800000003 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.699958+0000", 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1919000000000001e-05 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89695 8.8 8.d544e9a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 200.06559108100001, 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96264634999999998, 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:44:03.819 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89695 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737947+0000", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738028+0000", 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0275e-05 2024-09-13T14:44:03.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738048+0000", 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0489999999999999e-05 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738056+0000", 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1519999999999992e-06 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738110+0000", 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3384999999999998e-05 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738119+0000", 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9080000000000001e-06 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.700588+0000", 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96246986300000004 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.700596+0000", 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8140000000000001e-06 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89697 8.d 8.41b848dd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 200.06172790900001, 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95496279900000003, 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.824 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89697 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741811+0000", 2024-09-13T14:44:03.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741897+0000", 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6118000000000005e-05 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741917+0000", 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0013999999999998e-05 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741924+0000", 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.525e-06 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741979+0000", 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4194000000000003e-05 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:44:03.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741986+0000", 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7270000000000007e-06 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696767+0000", 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95478041400000002 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696776+0000", 2024-09-13T14:44:03.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1290000000000002e-06 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89699 8.d 8.b203ad2d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 200.05791663700001, 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95118529100000004, 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:44:03.830 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89699 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745623+0000", 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745687+0000", 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4147999999999996e-05 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745703+0000", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5985e-05 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745711+0000", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2900000000000002e-06 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745763+0000", 2024-09-13T14:44:03.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1915e-05 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745770+0000", 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.17e-06 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696804+0000", 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95103346799999999 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696810+0000", 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8499999999999999e-06 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89677 8.9 e318/317 8:975431c6:::benchmark_data_smithi017_253885_object89676:head v 318'5498, mlcod=0'0)", 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:45.983188+0000", 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.82035249099999, 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049527090000000003, 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983188+0000", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983188+0000", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983187+0000", 2024-09-13T14:44:03.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983225+0000", 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8047999999999999e-05 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983248+0000", 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2914000000000001e-05 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.983257+0000", 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7319999999999993e-06 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986106+0000", 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028482999999999998 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986134+0000", 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8787000000000001e-05 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032643+0000", 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046508163999999998 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032716+0000", 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2947000000000005e-05 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89693 8.9 e318/317 8:9608d7fe:::benchmark_data_smithi017_253885_object89692:head v 318'5499, mlcod=0'0)", 2024-09-13T14:44:03.839 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:45.994688+0000", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.80885248999999, 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.038055618999999999, 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994688+0000", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994688+0000", 2024-09-13T14:44:03.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994687+0000", 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994759+0000", 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2415000000000004e-05 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994783+0000", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3275999999999999e-05 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994792+0000", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7580000000000004e-06 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.995059+0000", 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00026702699999999998 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.995110+0000", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0278999999999998e-05 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032721+0000", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.037611644 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032744+0000", 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2799999999999999e-05 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.843 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89717 8.8 e318/317 8:1a565b38:::benchmark_data_smithi017_253885_object89716:head v 318'5512, mlcod=0'0)", 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.152650+0000", 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.65089099100001, 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031663869999999997, 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.152650+0000", 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.152650+0000", 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.152649+0000", 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.154060+0000", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014113280000000001 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.154075+0000", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4786e-05 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.154081+0000", 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7749999999999998e-06 2024-09-13T14:44:03.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.160795+0000", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0067141040000000003 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.160816+0000", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0945e-05 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.184198+0000", 2024-09-13T14:44:03.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023381575000000002 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.184314+0000", 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116196 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89734 8.0 e318/317 8:0997556a:::benchmark_data_smithi017_253885_object89733:head v 318'5635, mlcod=0'0)", 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.156645+0000", 2024-09-13T14:44:03.848 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.64689598699999, 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040731521, 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156645+0000", 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156645+0000", 2024-09-13T14:44:03.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156644+0000", 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156693+0000", 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9484000000000003e-05 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156711+0000", 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7737999999999999e-05 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.156718+0000", 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5030000000000002e-06 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.167618+0000", 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010900289000000001 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.167634+0000", 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6390999999999999e-05 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197330+0000", 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.029696120999999999 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197377+0000", 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6149999999999997e-05 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89742 8.0 e318/317 8:07c7a2c6:::benchmark_data_smithi017_253885_object89741:head v 318'5636, mlcod=0'0)", 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.159803+0000", 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.64373779600001, 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.037595358000000002, 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.853 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159803+0000", 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159803+0000", 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159802+0000", 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159857+0000", 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4661999999999999e-05 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159876+0000", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9008000000000002e-05 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.159885+0000", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6949999999999995e-06 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.172648+0000", 2024-09-13T14:44:03.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012762394 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.172673+0000", 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5446999999999999e-05 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197380+0000", 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024706961999999999 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197399+0000", 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8386999999999999e-05 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89743 8.8 e318/317 8:149edfa7:::benchmark_data_smithi017_253885_object89742:head v 318'5513, mlcod=0'0)", 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.163283+0000", 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.64025809099999, 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.051970978000000001, 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.858 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163283+0000", 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163283+0000", 2024-09-13T14:44:03.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163280+0000", 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163328+0000", 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7669e-05 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163353+0000", 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4819000000000001e-05 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163364+0000", 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0749e-05 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.178384+0000", 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015020742 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.178410+0000", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6118e-05 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215137+0000", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.036726101999999997 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215254+0000", 2024-09-13T14:44:03.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117283 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89747 8.0 e318/317 8:0eb65409:::benchmark_data_smithi017_253885_object89746:head v 318'5637, mlcod=0'0)", 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.163373+0000", 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.64016828600001, 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.051939649999999997, 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163373+0000", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163373+0000", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163372+0000", 2024-09-13T14:44:03.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163403+0000", 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1000000000000001e-05 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163414+0000", 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0689999999999999e-05 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.163419+0000", 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1479999999999997e-06 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.184197+0000", 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020778097999999998 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.184222+0000", 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5143e-05 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215261+0000", 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031038099999999999 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215312+0000", 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1768000000000003e-05 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89754 8.0 e318/317 8:01719362:::benchmark_data_smithi017_253885_object89753:head v 318'5638, mlcod=0'0)", 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.166433+0000", 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.63710796999999, 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048975484, 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166433+0000", 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166433+0000", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166432+0000", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166481+0000", 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9357000000000002e-05 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166502+0000", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1291999999999999e-05 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166511+0000", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9230000000000008e-06 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.190692+0000", 2024-09-13T14:44:03.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.02418091 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.190718+0000", 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5973e-05 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215368+0000", 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024649996 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215408+0000", 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0268000000000001e-05 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89758 8.8 e318/317 8:1143e5f0:::benchmark_data_smithi017_253885_object89757:head v 318'5514, mlcod=0'0)", 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.166520+0000", 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.63702054300001, 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048845539, 2024-09-13T14:44:03.873 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166520+0000", 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166520+0000", 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166520+0000", 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166553+0000", 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2743999999999997e-05 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166560+0000", 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8299999999999998e-06 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.166567+0000", 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8839999999999994e-06 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.190678+0000", 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024110782000000001 2024-09-13T14:44:03.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.190706+0000", 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7418000000000001e-05 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215316+0000", 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024610196000000001 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215366+0000", 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0099000000000003e-05 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89762 8.0 e318/317 8:01ff7021:::benchmark_data_smithi017_253885_object89761:head v 318'5639, mlcod=0'0)", 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.171289+0000", 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.632251919, 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.044163513000000001, 2024-09-13T14:44:03.878 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171289+0000", 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171289+0000", 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171288+0000", 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171368+0000", 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9831000000000002e-05 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171382+0000", 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3910000000000001e-05 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.171388+0000", 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9519999999999999e-06 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197421+0000", 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.026033595 2024-09-13T14:44:03.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.197445+0000", 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4315e-05 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215413+0000", 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017967371999999999 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.215453+0000", 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9765000000000003e-05 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89780 8.3 e318/317 8:c71bd7ce:::benchmark_data_smithi017_253885_object89779:head v 318'5473, mlcod=0'0)", 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.219213+0000", 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.58432768500001, 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.029089924, 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.883 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.219213+0000", 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.219213+0000", 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.219212+0000", 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220129+0000", 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00091694900000000002 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220152+0000", 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2957000000000001e-05 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220159+0000", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.498e-06 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220271+0000", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111988 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220292+0000", 2024-09-13T14:44:03.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1114000000000001e-05 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.248194+0000", 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.027902249 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.248303+0000", 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108729 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:89781 8.3 e318/317 8:c90c3d2b:::benchmark_data_smithi017_253885_object89780:head v 318'5474, mlcod=0'0)", 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:46.220173+0000", 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 197.58336764200001, 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.028182625999999999, 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220173+0000", 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220173+0000", 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220173+0000", 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220210+0000", 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7289000000000002e-05 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220217+0000", 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3849999999999999e-06 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.220222+0000", 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.989e-06 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.227435+0000", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072130459999999999 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.227463+0000", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8815000000000001e-05 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.248310+0000", 2024-09-13T14:44:03.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020846750000000001 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.248356+0000", 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5763000000000002e-05 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:91366 8.3 e323/317 8:c3dacf42:::benchmark_data_smithi017_253885_object25:head v 323'5583, mlcod=0'0)", 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:11.607863+0000", 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 172.19567752200001, 2024-09-13T14:44:03.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.029627707, 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607863+0000", 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607863+0000", 2024-09-13T14:44:03.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607862+0000", 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607875+0000", 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3453e-05 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607892+0000", 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6577000000000001e-05 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.607900+0000", 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5490000000000006e-06 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.612786+0000", 2024-09-13T14:44:03.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048858779999999997 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.612811+0000", 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4955000000000001e-05 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.637452+0000", 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024640406 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.637491+0000", 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9629999999999998e-05 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:91382 8.a e323/317 8:5798ee68:::benchmark_data_smithi017_253885_object41:head v 323'5780, mlcod=0'0)", 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:11.625275+0000", 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 172.17826616599999, 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030432794999999999, 2024-09-13T14:44:03.897 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625275+0000", 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625275+0000", 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625273+0000", 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625295+0000", 2024-09-13T14:44:03.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1965999999999999e-05 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625311+0000", 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5835e-05 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.625318+0000", 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5089999999999996e-06 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.642144+0000", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016825667999999998 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.642166+0000", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1841999999999999e-05 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.653565+0000", 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011398920999999999 2024-09-13T14:44:03.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.655708+0000", 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021428749999999998 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:91388 8.4 e323/317 8:2fbe8c86:::benchmark_data_smithi017_253885_object47:head v 323'5646, mlcod=0'0)", 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:11.668758+0000", 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 172.13478328400001, 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039699994000000002, 2024-09-13T14:44:03.902 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668758+0000", 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668758+0000", 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668757+0000", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668765+0000", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7860000000000002e-06 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668784+0000", 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9128e-05 2024-09-13T14:44:03.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.668791+0000", 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3720000000000001e-06 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.669206+0000", 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000414731 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.669230+0000", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4456000000000001e-05 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.708213+0000", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.038982676000000001 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.708458+0000", 2024-09-13T14:44:03.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000244952 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:91394 8.4 e323/317 8:211acc15:::benchmark_data_smithi017_253885_object53:head v 323'5647, mlcod=0'0)", 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:11.675541+0000", 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 172.128000407, 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.032966569000000001, 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:44:03.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675541+0000", 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675541+0000", 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:44:03.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675539+0000", 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675549+0000", 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9680000000000001e-06 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675570+0000", 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1169e-05 2024-09-13T14:44:03.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.675577+0000", 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7299999999999999e-06 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.678258+0000", 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002680647 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:44:03.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.678284+0000", 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6367999999999999e-05 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.708461+0000", 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030176583 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:11.708507+0000", 2024-09-13T14:44:03.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6480000000000002e-05 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:44:03.912 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:44:03.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:03.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:04.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:04.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:04.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:04.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:04.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:04.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:04.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:04.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:04 smithi017 ceph-mon[107556]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 39072/210414 objects degraded (18.569%); 4324/210414 objects misplaced (2.055%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:44:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 39072/210414 objects degraded (18.569%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:04 smithi017 ceph-mon[103001]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 39072/210414 objects degraded (18.569%); 4324/210414 objects misplaced (2.055%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:44:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 39072/210414 objects degraded (18.569%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:04 smithi189 ceph-mon[90992]: pgmap v1042: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 39072/210414 objects degraded (18.569%); 4324/210414 objects misplaced (2.055%); 915 KiB/s, 14 objects/s recovering 2024-09-13T14:44:04.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 39072/210414 objects degraded (18.569%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:04.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:04.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:04.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:04.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:04.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:44:04.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:04.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:04.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:04.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:05.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:05.042 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:05.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:05.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:05.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:05.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:05.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:05.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:05.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:05.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:05.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:05.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:05.668 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:05.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:05.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:05.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:05.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:05.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:06.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:06.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:06.147 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:06.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:06.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:06.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:06.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:06.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:06.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:06.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:06.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:06 smithi017 ceph-mon[103001]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39038/209337 objects degraded (18.648%); 4288/209337 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:44:06.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:06 smithi017 ceph-mon[107556]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39038/209337 objects degraded (18.648%); 4288/209337 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:44:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:06.654 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:06.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:06.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:06 smithi189 ceph-mon[90992]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 39038/209337 objects degraded (18.648%); 4288/209337 objects misplaced (2.048%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:44:06.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:06.800 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:06.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:06.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:07.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:07.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:07] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:44:07.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:07.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:07.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:07.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:08.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:08.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:08.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:08.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:08.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:08.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:08.316 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:08.316 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 39038/209337 objects degraded (18.648%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1444,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":345,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":69779,"data_bytes":4575824566,"bytes_used":17698209792,"bytes_avail":750262075392,"bytes_total":767960285184,"degraded_objects":39038,"degraded_total":209337,"degraded_ratio":0.18648399470709909,"misplaced_objects":4288,"misplaced_total":209337,"misplaced_ratio":0.020483717641888437,"recovering_objects_per_sec":13,"recovering_bytes_per_sec":917235,"recovering_keys_per_sec":0,"num_objects_recovered":140,"num_bytes_recovered":9175040,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":115},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 22s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:44:08.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:08.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:08.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2842252218' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38928/208728 objects degraded (18.650%); 4288/208728 objects misplaced (2.054%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2842252218' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38928/208728 objects degraded (18.650%); 4288/208728 objects misplaced (2.054%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2842252218' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: pgmap v1044: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38928/208728 objects degraded (18.650%); 4288/208728 objects misplaced (2.054%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:08.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:08.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:08.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:08.654 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:08.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:08.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:08.809 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:08.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:08.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:08.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:09.024 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:44:09.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:09.027 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:09.032 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:09.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:09.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:09.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:09.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:09.316 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:09.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38928/208728 objects degraded (18.650%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:09.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38928/208728 objects degraded (18.650%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:09.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:09.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:09.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38928/208728 objects degraded (18.650%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:09.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:09.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:09.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:09.818 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:09.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:09.943 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:10.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:10.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:10.127 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:44:10.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:10.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:10.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:10.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:10.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:10.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:10 smithi017 ceph-mon[107556]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38893/207696 objects degraded (18.726%); 4251/207696 objects misplaced (2.047%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:10 smithi017 ceph-mon[103001]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38893/207696 objects degraded (18.726%); 4251/207696 objects misplaced (2.047%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:10 smithi189 ceph-mon[90992]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38893/207696 objects degraded (18.726%); 4251/207696 objects misplaced (2.047%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:10.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:10.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:10.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:10.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:10.687 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:10.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:10.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:10.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:10.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:10.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.941 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:10.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:11.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:11.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:11.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:11.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:11.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:11.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:11.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:11.413 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[103001]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:44:11.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:11.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:11.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:11 smithi017 ceph-mon[107556]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:44:11.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:11.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:11 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:11.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2614722960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:11.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:11 smithi189 ceph-mon[90992]: osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:44:11.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:11.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:11.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:11.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:11.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:11.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:11.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:11.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:12.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:12.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:12.110 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:44:12.110 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:44:12.111 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:44:12.111 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:44:12.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:12.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:12.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:12.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:12.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:12.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:12 smithi017 ceph-mon[103001]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 38893/206565 objects degraded (18.828%); 4251/206565 objects misplaced (2.058%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:12 smithi017 ceph-mon[107556]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 38893/206565 objects degraded (18.828%); 4251/206565 objects misplaced (2.058%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:12.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:12 smithi189 ceph-mon[90992]: pgmap v1047: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 38893/206565 objects degraded (18.828%); 4251/206565 objects misplaced (2.058%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:12.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:12.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:12.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:12.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:12.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:12.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:12.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:12.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:12.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:13.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:13.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:13.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:13.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:13.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:13.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:13.699 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:13.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:13.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:13.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:13.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:13.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:14.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:14.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:14.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:14.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:14.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:14.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:14.329 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:14 smithi017 ceph-mon[103001]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38783/206019 objects degraded (18.825%); 4251/206019 objects misplaced (2.063%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:14.329 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38783/206019 objects degraded (18.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:14.330 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:14 smithi017 ceph-mon[107556]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38783/206019 objects degraded (18.825%); 4251/206019 objects misplaced (2.063%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:14.330 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38783/206019 objects degraded (18.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:14.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:14.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:14.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:14 smithi189 ceph-mon[90992]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38783/206019 objects degraded (18.825%); 4251/206019 objects misplaced (2.063%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38783/206019 objects degraded (18.825%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:14.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:14.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:14.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:14.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:14.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:15.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:15.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:15.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:15.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:15.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:15.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:15.300 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:15.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.311 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:15.311 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:15.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:15.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:15.330 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:15.346 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:15.347 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:15.362 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:15.362 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:15.378 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:15.379 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:15.394 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:15.394 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:15.411 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:15.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:15.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:15.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:15.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:15.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:15.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:15.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:44:15.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:15.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.924 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.946 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:16.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:16.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:16.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:16.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:44:16.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:16.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:16.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.421 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:16.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:16.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:16 smithi017 ceph-mon[103001]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 38743/205119 objects degraded (18.888%); 4213/205119 objects misplaced (2.054%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:44:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3776101782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:16 smithi017 ceph-mon[107556]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 38743/205119 objects degraded (18.888%); 4213/205119 objects misplaced (2.054%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:44:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3776101782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:16.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:16.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:16.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:16.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:16 smithi189 ceph-mon[90992]: pgmap v1049: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 38743/205119 objects degraded (18.888%); 4213/205119 objects misplaced (2.054%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:44:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3776101782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:16.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:16.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:16.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:16.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:17.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:17.153 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:17.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:17.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:17.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:17.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:17.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:17.371 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:17] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:44:17.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:17.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:17.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:17.701 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:17.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:17.825 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:17.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:17.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:18.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:18.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:18.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:18.300 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:18.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:18.423 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:18.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:18.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:18.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:18.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:18.586 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:18 smithi017 ceph-mon[103001]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 38657/204582 objects degraded (18.896%); 4213/204582 objects misplaced (2.059%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:18.587 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:18 smithi017 ceph-mon[107556]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 38657/204582 objects degraded (18.896%); 4213/204582 objects misplaced (2.059%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:18.675 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:44:18.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:18.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:18 smithi189 ceph-mon[90992]: pgmap v1050: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 38657/204582 objects degraded (18.896%); 4213/204582 objects misplaced (2.059%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:18.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:18.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:18.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:18.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:19.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:19.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:19.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:19.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:19.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:19.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:19.454 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:19.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:19.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:19.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38657/204582 objects degraded (18.896%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:19 smithi017 ceph-mon[103001]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38657/204582 objects degraded (18.896%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:19 smithi017 ceph-mon[107556]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:19.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38657/204582 objects degraded (18.896%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:19 smithi189 ceph-mon[90992]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:19.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:19.818 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1050,"stamp":"2024-09-13T14:44:17.203870+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4471950006,"num_objects":68194,"num_object_clones":0,"num_object_copies":204582,"num_objects_missing_on_primary":3261,"num_objects_missing":3261,"num_objects_degraded":38657,"num_objects_misplaced":4213,"num_objects_unfound":0,"num_objects_dirty":68194,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":115459,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22728,"num_bytes_recovered":1425508509,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116244,"ondisk_log_size":116244,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17010192,"kb_used_data":12034512,"kb_used_omap":44,"kb_used_meta":4975571,"kb_avail":732951024,"statfs":{"total":767960285184,"available":750541848576,"internally_reserved":0,"allocated":12323340288,"data_stored":12309000567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5094984951},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,0,2,2,4,1,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-90570752,"num_objects":-1382,"num_object_clones":0,"num_object_copies":-4146,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-271,"num_objects_misplaced":-75,"num_objects_unfound":0,"num_objects_dirty":-1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1382,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":173,"num_bytes_recovered":11337728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002805"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":575,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208983+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:44:10.208983+0000","last_peered":"2024-09-13T14:44:10.208983+0000","last_clean":"2024-09-13T14:44:10.208983+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:44:10.208983+0000","last_undegraded":"2024-09-13T14:44:10.208983+0000","last_fullsized":"2024-09-13T14:44:10.208983+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":344,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.291049+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:43:57.291049+0000","last_peered":"2024-09-13T14:43:57.291049+0000","last_clean":"2024-09-13T14:43:57.291049+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:43:57.291049+0000","last_undegraded":"2024-09-13T14:43:57.291049+0000","last_fullsized":"2024-09-13T14:43:57.291049+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280189+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:43:57.280189+0000","last_peered":"2024-09-13T14:43:57.280189+0000","last_clean":"2024-09-13T14:43:57.280189+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:43:57.280189+0000","last_undegraded":"2024-09-13T14:43:57.280189+0000","last_fullsized":"2024-09-13T14:43:57.280189+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":560,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.173954+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:44:10.173954+0000","last_peered":"2024-09-13T14:44:10.173954+0000","last_clean":"2024-09-13T14:44:10.173954+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:44:10.173954+0000","last_undegraded":"2024-09-13T14:44:10.173954+0000","last_fullsized":"2024-09-13T14:44:10.173954+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":561,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174239+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:44:10.174239+0000","last_peered":"2024-09-13T14:44:10.174239+0000","last_clean":"2024-09-13T14:44:10.174239+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:44:10.174239+0000","last_undegraded":"2024-09-13T14:44:10.174239+0000","last_fullsized":"2024-09-13T14:44:10.174239+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":313,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.290929+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:43:57.290929+0000","last_peered":"2024-09-13T14:43:57.290929+0000","last_clean":"2024-09-13T14:43:57.290929+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:43:57.290929+0000","last_undegraded":"2024-09-13T14:43:57.290929+0000","last_fullsized":"2024-09-13T14:43:57.290929+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":575,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.209242+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:44:10.209242+0000","last_peered":"2024-09-13T14:44:10.209242+0000","last_clean":"2024-09-13T14:44:10.209242+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:44:10.209242+0000","last_undegraded":"2024-09-13T14:44:10.209242+0000","last_fullsized":"2024-09-13T14:44:10.209242+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":335,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279516+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:43:57.279516+0000","last_peered":"2024-09-13T14:43:57.279516+0000","last_clean":"2024-09-13T14:43:57.279516+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:43:57.279516+0000","last_undegraded":"2024-09-13T14:43:57.279516+0000","last_fullsized":"2024-09-13T14:43:57.279516+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"346'7129","reported_seq":11673,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:16.610338+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:44:16.610338+0000","last_peered":"2024-09-13T14:44:16.610338+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:44:16.610338+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7129,"log_dups_size":0,"ondisk_log_size":7129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272695296,"num_objects":4161,"num_object_clones":0,"num_object_copies":12483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7237,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7129,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97255424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3177}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":339,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.291288+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:43:57.291288+0000","last_peered":"2024-09-13T14:43:57.291288+0000","last_clean":"2024-09-13T14:43:57.291288+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:43:57.291288+0000","last_undegraded":"2024-09-13T14:43:57.291288+0000","last_fullsized":"2024-09-13T14:43:57.291288+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":258,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.278984+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:43:57.278984+0000","last_peered":"2024-09-13T14:43:57.278984+0000","last_clean":"2024-09-13T14:43:57.278984+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:43:57.278984+0000","last_undegraded":"2024-09-13T14:43:57.278984+0000","last_fullsized":"2024-09-13T14:43:57.278984+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"346'7183","reported_seq":16726,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.850749+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:44:12.850749+0000","last_peered":"2024-09-13T14:44:12.850749+0000","last_clean":"2024-09-13T14:44:12.850749+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:44:12.850749+0000","last_undegraded":"2024-09-13T14:44:12.850749+0000","last_fullsized":"2024-09-13T14:44:12.850749+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7183,"log_dups_size":0,"ondisk_log_size":7183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282132480,"num_objects":4305,"num_object_clones":0,"num_object_copies":12915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7183,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":357,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.291509+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:43:57.291509+0000","last_peered":"2024-09-13T14:43:57.291509+0000","last_clean":"2024-09-13T14:43:57.291509+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:43:57.291509+0000","last_undegraded":"2024-09-13T14:43:57.291509+0000","last_fullsized":"2024-09-13T14:43:57.291509+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326423+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:43:57.326423+0000","last_peered":"2024-09-13T14:43:57.326423+0000","last_clean":"2024-09-13T14:43:57.326423+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:43:57.326423+0000","last_undegraded":"2024-09-13T14:43:57.326423+0000","last_fullsized":"2024-09-13T14:43:57.326423+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"346'7407","reported_seq":7506,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:13.898486+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:44:13.898486+0000","last_peered":"2024-09-13T14:44:13.898486+0000","last_clean":"2024-09-13T14:44:13.898486+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:44:13.898486+0000","last_undegraded":"2024-09-13T14:44:13.898486+0000","last_fullsized":"2024-09-13T14:44:13.898486+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287768576,"num_objects":4391,"num_object_clones":0,"num_object_copies":13173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7407,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":311,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.292923+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:43:57.292923+0000","last_peered":"2024-09-13T14:43:57.292923+0000","last_clean":"2024-09-13T14:43:57.292923+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:43:57.292923+0000","last_undegraded":"2024-09-13T14:43:57.292923+0000","last_fullsized":"2024-09-13T14:43:57.292923+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":326,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279748+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:43:57.279748+0000","last_peered":"2024-09-13T14:43:57.279748+0000","last_clean":"2024-09-13T14:43:57.279748+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:43:57.279748+0000","last_undegraded":"2024-09-13T14:43:57.279748+0000","last_fullsized":"2024-09-13T14:43:57.279748+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"346'7262","reported_seq":7361,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:14.741675+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:44:14.741675+0000","last_peered":"2024-09-13T14:44:14.741675+0000","last_clean":"2024-09-13T14:44:14.741675+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:44:14.741675+0000","last_undegraded":"2024-09-13T14:44:14.741675+0000","last_fullsized":"2024-09-13T14:44:14.741675+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7262,"log_dups_size":0,"ondisk_log_size":7262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7262,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":274,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280496+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:43:57.280496+0000","last_peered":"2024-09-13T14:43:57.280496+0000","last_clean":"2024-09-13T14:43:57.280496+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:43:57.280496+0000","last_undegraded":"2024-09-13T14:43:57.280496+0000","last_fullsized":"2024-09-13T14:43:57.280496+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":587,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174160+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:44:10.174160+0000","last_peered":"2024-09-13T14:44:10.174160+0000","last_clean":"2024-09-13T14:44:10.174160+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:44:10.174160+0000","last_undegraded":"2024-09-13T14:44:10.174160+0000","last_fullsized":"2024-09-13T14:44:10.174160+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"346'7186","reported_seq":7282,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:15.580266+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:44:15.580266+0000","last_peered":"2024-09-13T14:44:15.580266+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:44:15.580266+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280231936,"num_objects":4276,"num_object_clones":0,"num_object_copies":12828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7186,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1357}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":457,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.290737+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:43:57.290737+0000","last_peered":"2024-09-13T14:43:57.290737+0000","last_clean":"2024-09-13T14:43:57.290737+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:43:57.290737+0000","last_undegraded":"2024-09-13T14:43:57.290737+0000","last_fullsized":"2024-09-13T14:43:57.290737+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":585,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208993+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:44:10.208993+0000","last_peered":"2024-09-13T14:44:10.208993+0000","last_clean":"2024-09-13T14:44:10.208993+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:44:10.208993+0000","last_undegraded":"2024-09-13T14:44:10.208993+0000","last_fullsized":"2024-09-13T14:44:10.208993+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326761+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:43:57.326761+0000","last_peered":"2024-09-13T14:43:57.326761+0000","last_clean":"2024-09-13T14:43:57.326761+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:43:57.326761+0000","last_undegraded":"2024-09-13T14:43:57.326761+0000","last_fullsized":"2024-09-13T14:43:57.326761+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"346'6913","reported_seq":7012,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:14.497524+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:44:14.497524+0000","last_peered":"2024-09-13T14:44:14.497524+0000","last_clean":"2024-09-13T14:44:14.497524+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:44:14.497524+0000","last_undegraded":"2024-09-13T14:44:14.497524+0000","last_fullsized":"2024-09-13T14:44:14.497524+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6913,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":161,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279028+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:43:57.279028+0000","last_peered":"2024-09-13T14:43:57.279028+0000","last_clean":"2024-09-13T14:43:57.279028+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:43:57.279028+0000","last_undegraded":"2024-09-13T14:43:57.279028+0000","last_fullsized":"2024-09-13T14:43:57.279028+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":347,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279280+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:43:57.279280+0000","last_peered":"2024-09-13T14:43:57.279280+0000","last_clean":"2024-09-13T14:43:57.279280+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:43:57.279280+0000","last_undegraded":"2024-09-13T14:43:57.279280+0000","last_fullsized":"2024-09-13T14:43:57.279280+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":59,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.327050+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:43:57.327050+0000","last_peered":"2024-09-13T14:43:57.327050+0000","last_clean":"2024-09-13T14:43:57.327050+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:43:57.327050+0000","last_undegraded":"2024-09-13T14:43:57.327050+0000","last_fullsized":"2024-09-13T14:43:57.327050+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"346'7096","reported_seq":10973,"reported_epoch":346,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:13.865074+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:44:13.865074+0000","last_peered":"2024-09-13T14:44:13.865074+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:44:13.865074+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7096,"log_dups_size":0,"ondisk_log_size":7096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":3261,"num_objects_missing":3261,"num_objects_degraded":3636,"num_objects_misplaced":4213,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7096,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1390,"num_bytes_recovered":91095040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3261},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326249+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:43:57.326249+0000","last_peered":"2024-09-13T14:43:57.326249+0000","last_clean":"2024-09-13T14:43:57.326249+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:43:57.326249+0000","last_undegraded":"2024-09-13T14:43:57.326249+0000","last_fullsized":"2024-09-13T14:43:57.326249+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326977+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:43:57.326977+0000","last_peered":"2024-09-13T14:43:57.326977+0000","last_clean":"2024-09-13T14:43:57.326977+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:43:57.326977+0000","last_undegraded":"2024-09-13T14:43:57.326977+0000","last_fullsized":"2024-09-13T14:43:57.326977+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"346'6991","reported_seq":7090,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:14.728062+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:44:14.728062+0000","last_peered":"2024-09-13T14:44:14.728062+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:44:14.728062+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6991,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1299}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":154,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279550+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:43:57.279550+0000","last_peered":"2024-09-13T14:43:57.279550+0000","last_clean":"2024-09-13T14:43:57.279550+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:43:57.279550+0000","last_undegraded":"2024-09-13T14:43:57.279550+0000","last_fullsized":"2024-09-13T14:43:57.279550+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":577,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.214559+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:44:10.214559+0000","last_peered":"2024-09-13T14:44:10.214559+0000","last_clean":"2024-09-13T14:44:10.214559+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:44:10.214559+0000","last_undegraded":"2024-09-13T14:44:10.214559+0000","last_fullsized":"2024-09-13T14:44:10.214559+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":539,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129182+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:44:10.129182+0000","last_peered":"2024-09-13T14:44:10.129182+0000","last_clean":"2024-09-13T14:44:10.129182+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:44:10.129182+0000","last_undegraded":"2024-09-13T14:44:10.129182+0000","last_fullsized":"2024-09-13T14:44:10.129182+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"346'7025","reported_seq":7083,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:12.541084+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:44:12.541084+0000","last_peered":"2024-09-13T14:44:12.541084+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:44:12.541084+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7025,"log_dups_size":0,"ondisk_log_size":7025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274923520,"num_objects":4195,"num_object_clones":0,"num_object_copies":12585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7025,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1301}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":448,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.290875+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:43:57.290875+0000","last_peered":"2024-09-13T14:43:57.290875+0000","last_clean":"2024-09-13T14:43:57.290875+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:43:57.290875+0000","last_undegraded":"2024-09-13T14:43:57.290875+0000","last_fullsized":"2024-09-13T14:43:57.290875+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":870,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280395+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:43:57.280395+0000","last_peered":"2024-09-13T14:43:57.280395+0000","last_clean":"2024-09-13T14:43:57.280395+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:43:57.280395+0000","last_undegraded":"2024-09-13T14:43:57.280395+0000","last_fullsized":"2024-09-13T14:43:57.280395+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":530,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208626+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:44:10.208626+0000","last_peered":"2024-09-13T14:44:10.208626+0000","last_clean":"2024-09-13T14:44:10.208626+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:44:10.208626+0000","last_undegraded":"2024-09-13T14:44:10.208626+0000","last_fullsized":"2024-09-13T14:44:10.208626+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279503+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:43:57.279503+0000","last_peered":"2024-09-13T14:43:57.279503+0000","last_clean":"2024-09-13T14:43:57.279503+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:43:57.279503+0000","last_undegraded":"2024-09-13T14:43:57.279503+0000","last_fullsized":"2024-09-13T14:43:57.279503+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"346'7258","reported_seq":7355,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:14.734051+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:44:14.734051+0000","last_peered":"2024-09-13T14:44:14.734051+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:44:14.734051+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7258,"log_dups_size":0,"ondisk_log_size":7258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281149440,"num_objects":4290,"num_object_clones":0,"num_object_copies":12870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7258,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1390}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":384,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.131651+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:44:10.131651+0000","last_peered":"2024-09-13T14:44:10.131651+0000","last_clean":"2024-09-13T14:44:10.131651+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:44:10.131651+0000","last_undegraded":"2024-09-13T14:44:10.131651+0000","last_fullsized":"2024-09-13T14:44:10.131651+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":736,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.290801+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:43:57.290801+0000","last_peered":"2024-09-13T14:43:57.290801+0000","last_clean":"2024-09-13T14:43:57.290801+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:43:57.290801+0000","last_undegraded":"2024-09-13T14:43:57.290801+0000","last_fullsized":"2024-09-13T14:43:57.290801+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":579,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174049+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:44:10.174049+0000","last_peered":"2024-09-13T14:44:10.174049+0000","last_clean":"2024-09-13T14:44:10.174049+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:44:10.174049+0000","last_undegraded":"2024-09-13T14:44:10.174049+0000","last_fullsized":"2024-09-13T14:44:10.174049+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":584,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174317+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:44:10.174317+0000","last_peered":"2024-09-13T14:44:10.174317+0000","last_clean":"2024-09-13T14:44:10.174317+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:44:10.174317+0000","last_undegraded":"2024-09-13T14:44:10.174317+0000","last_fullsized":"2024-09-13T14:44:10.174317+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"346'7276","reported_seq":16916,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:14.696434+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:44:14.696434+0000","last_peered":"2024-09-13T14:44:14.696434+0000","last_clean":"2024-09-13T14:44:14.696434+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:44:14.696434+0000","last_undegraded":"2024-09-13T14:44:14.696434+0000","last_fullsized":"2024-09-13T14:44:14.696434+0000","mapping_epoch":330,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287571968,"num_objects":4388,"num_object_clones":0,"num_object_copies":13164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7276,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":477,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.290807+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:43:57.290807+0000","last_peered":"2024-09-13T14:43:57.290807+0000","last_clean":"2024-09-13T14:43:57.290807+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:43:57.290807+0000","last_undegraded":"2024-09-13T14:43:57.290807+0000","last_fullsized":"2024-09-13T14:43:57.290807+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":548,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208742+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:44:10.208742+0000","last_peered":"2024-09-13T14:44:10.208742+0000","last_clean":"2024-09-13T14:44:10.208742+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:44:10.208742+0000","last_undegraded":"2024-09-13T14:44:10.208742+0000","last_fullsized":"2024-09-13T14:44:10.208742+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":516,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128977+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:44:10.128977+0000","last_peered":"2024-09-13T14:44:10.128977+0000","last_clean":"2024-09-13T14:44:10.128977+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:44:10.128977+0000","last_undegraded":"2024-09-13T14:44:10.128977+0000","last_fullsized":"2024-09-13T14:44:10.128977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279991+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:43:57.279991+0000","last_peered":"2024-09-13T14:43:57.279991+0000","last_clean":"2024-09-13T14:43:57.279991+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:43:57.279991+0000","last_undegraded":"2024-09-13T14:43:57.279991+0000","last_fullsized":"2024-09-13T14:43:57.279991+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"346'7010","reported_seq":7108,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:15.285809+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:44:15.285809+0000","last_peered":"2024-09-13T14:44:15.285809+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:44:15.285809+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7010,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1289}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":154,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279804+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:43:57.279804+0000","last_peered":"2024-09-13T14:43:57.279804+0000","last_clean":"2024-09-13T14:43:57.279804+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:43:57.279804+0000","last_undegraded":"2024-09-13T14:43:57.279804+0000","last_fullsized":"2024-09-13T14:43:57.279804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":961,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279521+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:43:57.279521+0000","last_peered":"2024-09-13T14:43:57.279521+0000","last_clean":"2024-09-13T14:43:57.279521+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:43:57.279521+0000","last_undegraded":"2024-09-13T14:43:57.279521+0000","last_fullsized":"2024-09-13T14:43:57.279521+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326725+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:43:57.326725+0000","last_peered":"2024-09-13T14:43:57.326725+0000","last_clean":"2024-09-13T14:43:57.326725+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:43:57.326725+0000","last_undegraded":"2024-09-13T14:43:57.326725+0000","last_fullsized":"2024-09-13T14:43:57.326725+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":537,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129101+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:44:10.129101+0000","last_peered":"2024-09-13T14:44:10.129101+0000","last_clean":"2024-09-13T14:44:10.129101+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:44:10.129101+0000","last_undegraded":"2024-09-13T14:44:10.129101+0000","last_fullsized":"2024-09-13T14:44:10.129101+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"346'7121","reported_seq":16500,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:13.924939+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:44:13.924939+0000","last_peered":"2024-09-13T14:44:13.924939+0000","last_clean":"2024-09-13T14:44:13.924939+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:44:13.924939+0000","last_undegraded":"2024-09-13T14:44:13.924939+0000","last_fullsized":"2024-09-13T14:44:13.924939+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7121,"log_dups_size":0,"ondisk_log_size":7121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277807104,"num_objects":4239,"num_object_clones":0,"num_object_copies":12717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7121,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":345,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279858+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:43:57.279858+0000","last_peered":"2024-09-13T14:43:57.279858+0000","last_clean":"2024-09-13T14:43:57.279858+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:43:57.279858+0000","last_undegraded":"2024-09-13T14:43:57.279858+0000","last_fullsized":"2024-09-13T14:43:57.279858+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280073+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:43:57.280073+0000","last_peered":"2024-09-13T14:43:57.280073+0000","last_clean":"2024-09-13T14:43:57.280073+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:43:57.280073+0000","last_undegraded":"2024-09-13T14:43:57.280073+0000","last_fullsized":"2024-09-13T14:43:57.280073+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1116,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129286+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:44:10.129286+0000","last_peered":"2024-09-13T14:44:10.129286+0000","last_clean":"2024-09-13T14:44:10.129286+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:44:10.129286+0000","last_undegraded":"2024-09-13T14:44:10.129286+0000","last_fullsized":"2024-09-13T14:44:10.129286+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":518,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130875+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:44:10.130875+0000","last_peered":"2024-09-13T14:44:10.130875+0000","last_clean":"2024-09-13T14:44:10.130875+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:44:10.130875+0000","last_undegraded":"2024-09-13T14:44:10.130875+0000","last_fullsized":"2024-09-13T14:44:10.130875+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"346'7037","reported_seq":7136,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:15.564533+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:44:15.564533+0000","last_peered":"2024-09-13T14:44:15.564533+0000","last_clean":"2024-09-13T14:44:15.564533+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:44:15.564533+0000","last_undegraded":"2024-09-13T14:44:15.564533+0000","last_fullsized":"2024-09-13T14:44:15.564533+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7037,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":345,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279686+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:43:57.279686+0000","last_peered":"2024-09-13T14:43:57.279686+0000","last_clean":"2024-09-13T14:43:57.279686+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:43:57.279686+0000","last_undegraded":"2024-09-13T14:43:57.279686+0000","last_fullsized":"2024-09-13T14:43:57.279686+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":527,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129194+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:44:10.129194+0000","last_peered":"2024-09-13T14:44:10.129194+0000","last_clean":"2024-09-13T14:44:10.129194+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:44:10.129194+0000","last_undegraded":"2024-09-13T14:44:10.129194+0000","last_fullsized":"2024-09-13T14:44:10.129194+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":740,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.292901+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:43:57.292901+0000","last_peered":"2024-09-13T14:43:57.292901+0000","last_clean":"2024-09-13T14:43:57.292901+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:43:57.292901+0000","last_undegraded":"2024-09-13T14:43:57.292901+0000","last_fullsized":"2024-09-13T14:43:57.292901+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":562,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174116+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:44:10.174116+0000","last_peered":"2024-09-13T14:44:10.174116+0000","last_clean":"2024-09-13T14:44:10.174116+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:44:10.174116+0000","last_undegraded":"2024-09-13T14:44:10.174116+0000","last_fullsized":"2024-09-13T14:44:10.174116+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"346'7371","reported_seq":7470,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:13.738943+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:44:13.738943+0000","last_peered":"2024-09-13T14:44:13.738943+0000","last_clean":"2024-09-13T14:44:13.738943+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:44:13.738943+0000","last_undegraded":"2024-09-13T14:44:13.738943+0000","last_fullsized":"2024-09-13T14:44:13.738943+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283049984,"num_objects":4319,"num_object_clones":0,"num_object_copies":12957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7371,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326993+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:43:57.326993+0000","last_peered":"2024-09-13T14:43:57.326993+0000","last_clean":"2024-09-13T14:43:57.326993+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:43:57.326993+0000","last_undegraded":"2024-09-13T14:43:57.326993+0000","last_fullsized":"2024-09-13T14:43:57.326993+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":563,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.209406+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:44:10.209406+0000","last_peered":"2024-09-13T14:44:10.209406+0000","last_clean":"2024-09-13T14:44:10.209406+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:44:10.209406+0000","last_undegraded":"2024-09-13T14:44:10.209406+0000","last_fullsized":"2024-09-13T14:44:10.209406+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":903,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129780+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:44:10.129780+0000","last_peered":"2024-09-13T14:44:10.129780+0000","last_clean":"2024-09-13T14:44:10.129780+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:44:10.129780+0000","last_undegraded":"2024-09-13T14:44:10.129780+0000","last_fullsized":"2024-09-13T14:44:10.129780+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":561,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.214466+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:44:10.214466+0000","last_peered":"2024-09-13T14:44:10.214466+0000","last_clean":"2024-09-13T14:44:10.214466+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:44:10.214466+0000","last_undegraded":"2024-09-13T14:44:10.214466+0000","last_fullsized":"2024-09-13T14:44:10.214466+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"346'7247","reported_seq":18604,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.615173+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:44:12.615173+0000","last_peered":"2024-09-13T14:44:12.615173+0000","last_clean":"2024-09-13T14:44:12.615173+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:44:12.615173+0000","last_undegraded":"2024-09-13T14:44:12.615173+0000","last_fullsized":"2024-09-13T14:44:12.615173+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7247,"log_dups_size":0,"ondisk_log_size":7247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279576600,"num_objects":4267,"num_object_clones":0,"num_object_copies":12801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4267,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7247,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":414,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208780+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:44:10.208780+0000","last_peered":"2024-09-13T14:44:10.208780+0000","last_clean":"2024-09-13T14:44:10.208780+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:44:10.208780+0000","last_undegraded":"2024-09-13T14:44:10.208780+0000","last_fullsized":"2024-09-13T14:44:10.208780+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":598,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174217+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:44:10.174217+0000","last_peered":"2024-09-13T14:44:10.174217+0000","last_clean":"2024-09-13T14:44:10.174217+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:44:10.174217+0000","last_undegraded":"2024-09-13T14:44:10.174217+0000","last_fullsized":"2024-09-13T14:44:10.174217+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":764,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279911+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:43:57.279911+0000","last_peered":"2024-09-13T14:43:57.279911+0000","last_clean":"2024-09-13T14:43:57.279911+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:43:57.279911+0000","last_undegraded":"2024-09-13T14:43:57.279911+0000","last_fullsized":"2024-09-13T14:43:57.279911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":59,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326350+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:43:57.326350+0000","last_peered":"2024-09-13T14:43:57.326350+0000","last_clean":"2024-09-13T14:43:57.326350+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:43:57.326350+0000","last_undegraded":"2024-09-13T14:43:57.326350+0000","last_fullsized":"2024-09-13T14:43:57.326350+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326764+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:43:57.326764+0000","last_peered":"2024-09-13T14:43:57.326764+0000","last_clean":"2024-09-13T14:43:57.326764+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:43:57.326764+0000","last_undegraded":"2024-09-13T14:43:57.326764+0000","last_fullsized":"2024-09-13T14:43:57.326764+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":55,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326680+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:43:57.326680+0000","last_peered":"2024-09-13T14:43:57.326680+0000","last_clean":"2024-09-13T14:43:57.326680+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:43:57.326680+0000","last_undegraded":"2024-09-13T14:43:57.326680+0000","last_fullsized":"2024-09-13T14:43:57.326680+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":515,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130753+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:44:10.130753+0000","last_peered":"2024-09-13T14:44:10.130753+0000","last_clean":"2024-09-13T14:44:10.130753+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:44:10.130753+0000","last_undegraded":"2024-09-13T14:44:10.130753+0000","last_fullsized":"2024-09-13T14:44:10.130753+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":345,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.292843+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:43:57.292843+0000","last_peered":"2024-09-13T14:43:57.292843+0000","last_clean":"2024-09-13T14:43:57.292843+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:43:57.292843+0000","last_undegraded":"2024-09-13T14:43:57.292843+0000","last_fullsized":"2024-09-13T14:43:57.292843+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":311,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279109+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:43:57.279109+0000","last_peered":"2024-09-13T14:43:57.279109+0000","last_clean":"2024-09-13T14:43:57.279109+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:43:57.279109+0000","last_undegraded":"2024-09-13T14:43:57.279109+0000","last_fullsized":"2024-09-13T14:43:57.279109+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":519,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208540+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:44:10.208540+0000","last_peered":"2024-09-13T14:44:10.208540+0000","last_clean":"2024-09-13T14:44:10.208540+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:44:10.208540+0000","last_undegraded":"2024-09-13T14:44:10.208540+0000","last_fullsized":"2024-09-13T14:44:10.208540+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":571,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174157+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:44:10.174157+0000","last_peered":"2024-09-13T14:44:10.174157+0000","last_clean":"2024-09-13T14:44:10.174157+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:44:10.174157+0000","last_undegraded":"2024-09-13T14:44:10.174157+0000","last_fullsized":"2024-09-13T14:44:10.174157+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":575,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.214396+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:44:10.214396+0000","last_peered":"2024-09-13T14:44:10.214396+0000","last_clean":"2024-09-13T14:44:10.214396+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:44:10.214396+0000","last_undegraded":"2024-09-13T14:44:10.214396+0000","last_fullsized":"2024-09-13T14:44:10.214396+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":343,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.279865+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:43:57.279865+0000","last_peered":"2024-09-13T14:43:57.279865+0000","last_clean":"2024-09-13T14:43:57.279865+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:43:57.279865+0000","last_undegraded":"2024-09-13T14:43:57.279865+0000","last_fullsized":"2024-09-13T14:43:57.279865+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":593,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.214505+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:44:10.214505+0000","last_peered":"2024-09-13T14:44:10.214505+0000","last_clean":"2024-09-13T14:44:10.214505+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:44:10.214505+0000","last_undegraded":"2024-09-13T14:44:10.214505+0000","last_fullsized":"2024-09-13T14:44:10.214505+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":563,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174299+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:44:10.174299+0000","last_peered":"2024-09-13T14:44:10.174299+0000","last_clean":"2024-09-13T14:44:10.174299+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:44:10.174299+0000","last_undegraded":"2024-09-13T14:44:10.174299+0000","last_fullsized":"2024-09-13T14:44:10.174299+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":539,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130118+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:44:10.130118+0000","last_peered":"2024-09-13T14:44:10.130118+0000","last_clean":"2024-09-13T14:44:10.130118+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:44:10.130118+0000","last_undegraded":"2024-09-13T14:44:10.130118+0000","last_fullsized":"2024-09-13T14:44:10.130118+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":589,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208853+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:44:10.208853+0000","last_peered":"2024-09-13T14:44:10.208853+0000","last_clean":"2024-09-13T14:44:10.208853+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:44:10.208853+0000","last_undegraded":"2024-09-13T14:44:10.208853+0000","last_fullsized":"2024-09-13T14:44:10.208853+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":44,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.331493+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:43:57.331493+0000","last_peered":"2024-09-13T14:43:57.331493+0000","last_clean":"2024-09-13T14:43:57.331493+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:43:57.331493+0000","last_undegraded":"2024-09-13T14:43:57.331493+0000","last_fullsized":"2024-09-13T14:43:57.331493+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":485,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129417+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:44:10.129417+0000","last_peered":"2024-09-13T14:44:10.129417+0000","last_clean":"2024-09-13T14:44:10.129417+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:44:10.129417+0000","last_undegraded":"2024-09-13T14:44:10.129417+0000","last_fullsized":"2024-09-13T14:44:10.129417+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":567,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174051+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:44:10.174051+0000","last_peered":"2024-09-13T14:44:10.174051+0000","last_clean":"2024-09-13T14:44:10.174051+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:44:10.174051+0000","last_undegraded":"2024-09-13T14:44:10.174051+0000","last_fullsized":"2024-09-13T14:44:10.174051+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326687+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:43:57.326687+0000","last_peered":"2024-09-13T14:43:57.326687+0000","last_clean":"2024-09-13T14:43:57.326687+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:43:57.326687+0000","last_undegraded":"2024-09-13T14:43:57.326687+0000","last_fullsized":"2024-09-13T14:43:57.326687+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":311,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280572+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:43:57.280572+0000","last_peered":"2024-09-13T14:43:57.280572+0000","last_clean":"2024-09-13T14:43:57.280572+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:43:57.280572+0000","last_undegraded":"2024-09-13T14:43:57.280572+0000","last_fullsized":"2024-09-13T14:43:57.280572+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.331440+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:43:57.331440+0000","last_peered":"2024-09-13T14:43:57.331440+0000","last_clean":"2024-09-13T14:43:57.331440+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:43:57.331440+0000","last_undegraded":"2024-09-13T14:43:57.331440+0000","last_fullsized":"2024-09-13T14:43:57.331440+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":277,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.280284+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:43:57.280284+0000","last_peered":"2024-09-13T14:43:57.280284+0000","last_clean":"2024-09-13T14:43:57.280284+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:43:57.280284+0000","last_undegraded":"2024-09-13T14:43:57.280284+0000","last_fullsized":"2024-09-13T14:43:57.280284+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":585,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.208785+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:44:10.208785+0000","last_peered":"2024-09-13T14:44:10.208785+0000","last_clean":"2024-09-13T14:44:10.208785+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:44:10.208785+0000","last_undegraded":"2024-09-13T14:44:10.208785+0000","last_fullsized":"2024-09-13T14:44:10.208785+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":572,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.209084+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:44:10.209084+0000","last_peered":"2024-09-13T14:44:10.209084+0000","last_clean":"2024-09-13T14:44:10.209084+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:44:10.209084+0000","last_undegraded":"2024-09-13T14:44:10.209084+0000","last_fullsized":"2024-09-13T14:44:10.209084+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":45,"reported_epoch":345,"state":"active+clean","last_fresh":"2024-09-13T14:43:57.326235+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:43:57.326235+0000","last_peered":"2024-09-13T14:43:57.326235+0000","last_clean":"2024-09-13T14:43:57.326235+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:43:57.326235+0000","last_undegraded":"2024-09-13T14:43:57.326235+0000","last_fullsized":"2024-09-13T14:43:57.326235+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":576,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.174169+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:44:10.174169+0000","last_peered":"2024-09-13T14:44:10.174169+0000","last_clean":"2024-09-13T14:44:10.174169+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:44:10.174169+0000","last_undegraded":"2024-09-13T14:44:10.174169+0000","last_fullsized":"2024-09-13T14:44:10.174169+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4467261464,"num_objects":68166,"num_object_clones":0,"num_object_copies":204498,"num_objects_missing_on_primary":3261,"num_objects_missing":3261,"num_objects_degraded":38657,"num_objects_misplaced":4213,"num_objects_unfound":0,"num_objects_dirty":68166,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114512,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22671,"num_bytes_recovered":1425342464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12266319872,"data_stored":12266307656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114112,"ondisk_log_size":114112,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397897,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2423068,"kb_used_data":1522900,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91322084,"statfs":{"total":95995035648,"available":93513814016,"internally_reserved":0,"allocated":1559449600,"data_stored":1557641579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594125,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2326216,"kb_used_data":1728000,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":91418936,"statfs":{"total":95995035648,"available":93612990464,"internally_reserved":0,"allocated":1769472000,"data_stored":1767683733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665579,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649096,"kb_used_data":324864,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93096056,"statfs":{"total":95995035648,"available":95330361344,"internally_reserved":0,"allocated":332660736,"data_stored":330868800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986581,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1860692,"kb_used_data":965004,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":91884460,"statfs":{"total":95995035648,"available":94089687040,"internally_reserved":0,"allocated":988164096,"data_stored":986364173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,0,2,2,4,1,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697141,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2723268,"kb_used_data":2005372,"kb_used_omap":4,"kb_used_meta":717883,"kb_avail":91021884,"statfs":{"total":95995035648,"available":93206409216,"internally_reserved":0,"allocated":2053500928,"data_stored":2051713145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":735112267},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860668,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2541324,"kb_used_data":1924228,"kb_used_omap":8,"kb_used_meta":617079,"kb_avail":91203828,"statfs":{"total":95995035648,"available":93392719872,"internally_reserved":0,"allocated":1970409472,"data_stored":1968614035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":631889740},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020339,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2217600,"kb_used_data":1914040,"kb_used_omap":9,"kb_used_meta":303542,"kb_avail":91527552,"statfs":{"total":95995035648,"available":93724213248,"internally_reserved":0,"allocated":1959976960,"data_stored":1958192650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":310827845},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187729,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2268928,"kb_used_data":1650104,"kb_used_omap":7,"kb_used_meta":618808,"kb_avail":91476224,"statfs":{"total":95995035648,"available":93671653376,"internally_reserved":0,"allocated":1689706496,"data_stored":1687922452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":633659723},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1680015360,"data_stored":1680015360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1950482432,"data_stored":1950482432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1965690880,"data_stored":1965686808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2047082496,"data_stored":2047082496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":981073920,"data_stored":981073920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327421952,"data_stored":327417880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1764884480,"data_stored":1764884480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1549668352,"data_stored":1549664280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:19.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:19.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:19.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:19.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.280 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:20.281 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:20.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:20.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:20.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:20 smithi017 ceph-mon[103001]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38616/203691 objects degraded (18.958%); 4174/203691 objects misplaced (2.049%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:20 smithi017 ceph-mon[107556]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38616/203691 objects degraded (18.958%); 4174/203691 objects misplaced (2.049%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:20.483 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:20.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:20 smithi189 ceph-mon[90992]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 38616/203691 objects degraded (18.958%); 4174/203691 objects misplaced (2.049%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:20.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:20.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:20.674 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:20.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:20.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:20.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:20.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:44:20.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:20.946 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:20.951 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:21.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:21.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:21.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:21.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:21.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:21.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:21.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:21.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:21.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:21.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:21.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:21.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:21.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:21.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:21.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:21.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:21.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:21.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:22.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:22.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:22.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:22.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:22.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:22.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:22 smithi017 ceph-mon[103001]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38616/202746 objects degraded (19.046%); 4174/202746 objects misplaced (2.059%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:44:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:22 smithi017 ceph-mon[107556]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38616/202746 objects degraded (19.046%); 4174/202746 objects misplaced (2.059%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:44:22.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:22.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:22.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:22 smithi189 ceph-mon[90992]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38616/202746 objects degraded (19.046%); 4174/202746 objects misplaced (2.059%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:44:22.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:22.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:22.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:22.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:22.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:22.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:23.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:23.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:23.182 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:23.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:23.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:23.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:23.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:23.447 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:23.468 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:23.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:23.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:23.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:23.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:23.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:23.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:23.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:23.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:24.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:24.079 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:24.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:24.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:24.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:24.492 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:24 smithi017 ceph-mon[103001]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38530/202095 objects degraded (19.065%); 4174/202095 objects misplaced (2.065%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:24.492 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38530/202095 objects degraded (19.065%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:24.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:24 smithi017 ceph-mon[107556]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38530/202095 objects degraded (19.065%); 4174/202095 objects misplaced (2.065%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:24.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38530/202095 objects degraded (19.065%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:24.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:24.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:24.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:24 smithi189 ceph-mon[90992]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 38530/202095 objects degraded (19.065%); 4174/202095 objects misplaced (2.065%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:44:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38530/202095 objects degraded (19.065%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:24.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:24.792 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:24.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:24.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:24.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:24.826 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:24.843 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:24.843 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:24.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:24.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:24.880 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:24.880 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:24.897 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:24.897 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:24.913 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:24.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:24.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:24.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:25.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:25.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:25.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:25.279 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:44:25.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:25.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:25.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:25.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:25.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3335998134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3507254255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:25.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3335998134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3507254255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:25.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:25.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:25.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:25.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3335998134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3507254255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:25.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:25.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:25.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:26.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:26.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:26.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:26.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:26.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:26.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:26.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:26.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:26.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:26.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:26.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[107556]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 38493/200862 objects degraded (19.164%); 4136/200862 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[107556]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[103001]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 38493/200862 objects degraded (19.164%); 4136/200862 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:26 smithi017 ceph-mon[103001]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:44:26.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:26 smithi189 ceph-mon[90992]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 38493/200862 objects degraded (19.164%); 4136/200862 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:26 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:26 smithi189 ceph-mon[90992]: osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:44:26.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:26.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:26.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:26.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:26.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:27.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:27.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:27.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:27.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:27.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:27.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:27.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:27.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:27.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:27.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:27] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:44:27.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:27.810 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:27.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:27.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:27.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:28.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[103001]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38405/200142 objects degraded (19.189%); 4136/200142 objects misplaced (2.067%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:28 smithi017 ceph-mon[107556]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38405/200142 objects degraded (19.189%); 4136/200142 objects misplaced (2.067%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:28.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:28.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:28.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:28.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:28.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:28 smithi189 ceph-mon[90992]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38405/200142 objects degraded (19.189%); 4136/200142 objects misplaced (2.067%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:28.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:28.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:28.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:28.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:28.709 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:28.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:44:28.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:28.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:28.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:28.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:29.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:29.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:29.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:29.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:29.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:29.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:29.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:29.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38405/200142 objects degraded (19.189%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:29.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38405/200142 objects degraded (19.189%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:29.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:29.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38405/200142 objects degraded (19.189%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:29.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:29.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:29.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:29.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:29.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:29.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:29.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:29.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:29.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:30.042 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:30.043 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:44:30.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:30.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:82842 8.e 8:78b86351:::benchmark_data_smithi017_253885_object82841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:44:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 234.252896467, 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.122549702, 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82842 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:44:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881958+0000", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.33e-06 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882008+0000", 2024-09-13T14:44:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0303999999999998e-05 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882020+0000", 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1847e-05 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882027+0000", 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0389999999999999e-06 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984746+0000", 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10271846599999999 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984909+0000", 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016386699999999999 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.985058+0000", 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000148479 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.991823+0000", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067648980000000001 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996312+0000", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044889270000000002 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996354+0000", 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1857999999999998e-05 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996415+0000", 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1398999999999999e-05 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.004506+0000", 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080912880000000003 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(8.2 314/311 [PushReplyOp(8:4757a975:::benchmark_data_smithi017_253885_object3344:head)])", 2024-09-13T14:44:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 233.93178383599999, 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14042681400000001, 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203070+0000", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9399999999999993e-07 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203077+0000", 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6320000000000002e-06 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203093+0000", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6152000000000001e-05 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203102+0000", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999998e-06 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.334767+0000", 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13166481199999999 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.343496+0000", 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087293370000000002 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88690 8.e 8:774d5387:::benchmark_data_smithi017_253885_object88689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 227.65709691199999, 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13302814199999999, 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88690 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:44:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477757+0000", 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1900000000000001e-07 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477779+0000", 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2248000000000001e-05 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477791+0000", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1515000000000001e-05 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477799+0000", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8800000000000008e-06 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590385+0000", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112586266 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590547+0000", 2024-09-13T14:44:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162399 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590706+0000", 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015852800000000001 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607019+0000", 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016312884999999999 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607063+0000", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4583000000000003e-05 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607112+0000", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8779000000000002e-05 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607171+0000", 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8609000000000001e-05 2024-09-13T14:44:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610784+0000", 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003613631 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88698 8.e 8:7f48e2e9:::benchmark_data_smithi017_253885_object88697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 227.645456637, 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.125047359, 2024-09-13T14:44:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88698 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489398+0000", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5430000000000001e-06 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489430+0000", 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1897000000000003e-05 2024-09-13T14:44:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489443+0000", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2904999999999999e-05 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571286+0000", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081843369999999999 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.594934+0000", 2024-09-13T14:44:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023647714 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595081+0000", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014731100000000001 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595237+0000", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015634899999999999 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607076+0000", 2024-09-13T14:44:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011838987 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610972+0000", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038959049999999999 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611023+0000", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0523999999999999e-05 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611085+0000", 2024-09-13T14:44:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2390000000000004e-05 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.614444+0000", 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033584639999999998 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89686 8.2 8.4ba11582 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.41114705300001, 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97132002200000001, 2024-09-13T14:44:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89686 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723708+0000", 2024-09-13T14:44:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.531e-06 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723735+0000", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.741e-05 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723752+0000", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7365000000000001e-05 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723760+0000", 2024-09-13T14:44:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1240000000000005e-06 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723831+0000", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0325000000000002e-05 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723842+0000", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1249e-05 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695013+0000", 2024-09-13T14:44:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97117079500000003 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695026+0000", 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3223e-05 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:95761 8.1 e327/303 8:8304b12a:::benchmark_data_smithi017_253885_object4420:head v 327'5803, mlcod=327'5801)", 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 174.38425517600001, 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15775900900000001, 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750599+0000", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2399999999999997e-07 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750605+0000", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9750000000000004e-06 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750613+0000", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2930000000000001e-06 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901650+0000", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15103724800000001 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901711+0000", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1179999999999994e-05 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901732+0000", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0166999999999999e-05 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908314+0000", 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065826890000000001 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908357+0000", 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2633000000000003e-05 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:95762 8.c 8:352993b8:::benchmark_data_smithi017_253885_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 174.38059615399999, 2024-09-13T14:44:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.154645952, 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95762 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754258+0000", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5460000000000001e-06 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754265+0000", 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2899999999999999e-06 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901433+0000", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.147168615 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901452+0000", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8652999999999999e-05 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901526+0000", 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3640999999999995e-05 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901644+0000", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118359 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901746+0000", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010203400000000001 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.907274+0000", 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005528112 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908552+0000", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012774419999999999 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908783+0000", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023154999999999999 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908846+0000", 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3323000000000003e-05 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908903+0000", 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6387000000000001e-05 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104233 8.e 8:7e2470cd:::benchmark_data_smithi017_253885_object12892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.310673112, 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83912028100000002, 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104233 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824181+0000", 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2619999999999999e-06 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824186+0000", 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9359999999999998e-06 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824199+0000", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2951e-05 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824205+0000", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5690000000000004e-06 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824257+0000", 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2454000000000003e-05 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824265+0000", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0660000000000004e-06 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656221+0000", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83195581900000004 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656329+0000", 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010786200000000001 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656420+0000", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0880999999999999e-05 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663066+0000", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066462250000000004 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663119+0000", 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2852999999999997e-05 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663181+0000", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2327000000000001e-05 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663246+0000", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5300000000000002e-05 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663300+0000", 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3776000000000001e-05 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104242 8.e 8:7779217b:::benchmark_data_smithi017_253885_object12901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.09175527399999, 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.62049879299999999, 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104242 2024-09-13T14:44:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043100+0000", 2024-09-13T14:44:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7519999999999999e-06 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043107+0000", 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6920000000000002e-06 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043127+0000", 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9409e-05 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043134+0000", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4070000000000004e-06 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043204+0000", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0451999999999996e-05 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043215+0000", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0719e-05 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659704+0000", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.61648930499999999 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659805+0000", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010100200000000001 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659923+0000", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011727600000000001 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663438+0000", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035150530000000002 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663495+0000", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7058999999999997e-05 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663510+0000", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5038999999999999e-05 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663557+0000", 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6634000000000001e-05 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663597+0000", 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9993999999999997e-05 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104296 8.e 8:77431237:::benchmark_data_smithi017_253885_object12955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:44:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 100.68259042699999, 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.220321039, 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104296 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452264+0000", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.677e-06 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452271+0000", 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6869999999999997e-06 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452289+0000", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7584e-05 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452296+0000", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4390000000000003e-06 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452356+0000", 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0322999999999998e-05 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452365+0000", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3410000000000003e-06 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663647+0000", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21128269 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663780+0000", 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133106 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.664086+0000", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030593599999999998 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672353+0000", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008266509 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672411+0000", 2024-09-13T14:44:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8459999999999999e-05 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672453+0000", 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1813000000000001e-05 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672526+0000", 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2769999999999996e-05 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672584+0000", 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7704000000000002e-05 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104299 8.e 8:7d6c2a47:::benchmark_data_smithi017_253885_object12958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 100.671088325, 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.208997076, 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104299 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463766+0000", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1620000000000001e-06 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463771+0000", 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.865e-06 2024-09-13T14:44:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463782+0000", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.128e-05 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463787+0000", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4020000000000001e-06 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463855+0000", 2024-09-13T14:44:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7737999999999994e-05 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463868+0000", 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2928999999999999e-05 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667607+0000", 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.203738481 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667720+0000", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113057 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667828+0000", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108084 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672399+0000", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045716810000000002 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672597+0000", 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197944 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672659+0000", 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1593999999999993e-05 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672724+0000", 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5009e-05 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672762+0000", 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7849999999999998e-05 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113171 8.e 8:743a24ae:::benchmark_data_smithi017_253885_object21830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:04.828732+0000", 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.306121520000001, 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076144079999999996, 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113171 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828732+0000", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828732+0000", 2024-09-13T14:44:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828733+0000", 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2610000000000001e-06 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828738+0000", 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.164e-06 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828751+0000", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3040999999999999e-05 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828759+0000", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8709999999999997e-06 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828813+0000", 2024-09-13T14:44:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3919999999999999e-05 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.828949+0000", 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013589 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.829070+0000", 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121561 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.832989+0000", 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039187229999999998 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.833056+0000", 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7451000000000002e-05 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.836280+0000", 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032231930000000001 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.836318+0000", 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8009999999999997e-05 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.836346+0000", 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8323e-05 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113777 8.c 8:3dc4e0a8:::benchmark_data_smithi017_253885_object22436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.623904+0000", 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.510949461999999, 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077470359999999997, 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113777 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623904+0000", 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623904+0000", 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623904+0000", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3699999999999999e-07 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623908+0000", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9999999999999998e-06 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623921+0000", 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2543e-05 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623928+0000", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1380000000000001e-06 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623994+0000", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6192999999999995e-05 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.624080+0000", 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5715000000000002e-05 2024-09-13T14:44:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.624154+0000", 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4099999999999999e-05 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.631464+0000", 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007309467 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.631537+0000", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3714999999999995e-05 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.631558+0000", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0418999999999999e-05 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.631612+0000", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3844999999999997e-05 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.631651+0000", 2024-09-13T14:44:30.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9063999999999997e-05 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113784 8.6 8:6ff69c18:::benchmark_data_smithi017_253885_object22443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.639932+0000", 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.494921473000002, 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082839899999999998, 2024-09-13T14:44:30.215 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113784 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639932+0000", 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639932+0000", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639933+0000", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8250000000000001e-06 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639941+0000", 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0150000000000003e-06 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639962+0000", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0852999999999998e-05 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.639971+0000", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8419999999999994e-06 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.640049+0000", 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7724000000000003e-05 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.640179+0000", 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013019400000000001 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.640266+0000", 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7175000000000005e-05 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.646386+0000", 2024-09-13T14:44:30.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061197730000000002 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.646401+0000", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.52e-05 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648124+0000", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017228339999999999 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648174+0000", 2024-09-13T14:44:30.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9969999999999998e-05 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648216+0000", 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1585000000000002e-05 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113787 8.c 8:3ba09e56:::benchmark_data_smithi017_253885_object22446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.647814+0000", 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.487038636000001, 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008325987, 2024-09-13T14:44:30.222 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113787 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647814+0000", 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647814+0000", 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647816+0000", 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4389999999999999e-06 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647822+0000", 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7080000000000002e-06 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647835+0000", 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3363999999999999e-05 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647842+0000", 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9480000000000002e-06 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647899+0000", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7346000000000003e-05 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.647979+0000", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9562999999999997e-05 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648081+0000", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102398 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.654706+0000", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066249120000000002 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.654817+0000", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011108 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656032+0000", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012152770000000001 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656096+0000", 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3538000000000001e-05 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656140+0000", 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4413999999999999e-05 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113788 8.c 8:37659b22:::benchmark_data_smithi017_253885_object22447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.228 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.648331+0000", 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.486522355999998, 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079108640000000001, 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113788 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648331+0000", 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648331+0000", 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648332+0000", 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6299999999999993e-07 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648336+0000", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7299999999999996e-06 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648348+0000", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1897e-05 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.648361+0000", 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2605e-05 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.651445+0000", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030836539999999999 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.651534+0000", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9458999999999998e-05 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.651643+0000", 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109319 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.654838+0000", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031943219999999999 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.654937+0000", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9209999999999994e-05 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656191+0000", 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001254479 2024-09-13T14:44:30.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656219+0000", 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7398000000000001e-05 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.656242+0000", 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2827999999999999e-05 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113812 8.c 8:34df6e4f:::benchmark_data_smithi017_253885_object22471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.234 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.696871+0000", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.437981882999999, 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076698360000000002, 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113812 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696871+0000", 2024-09-13T14:44:30.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696871+0000", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696872+0000", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7800000000000001e-07 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696886+0000", 2024-09-13T14:44:30.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3953e-05 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696900+0000", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3638999999999999e-05 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696905+0000", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6189999999999999e-06 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.696954+0000", 2024-09-13T14:44:30.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8853000000000001e-05 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.697031+0000", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6557000000000006e-05 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.697090+0000", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9818000000000002e-05 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.700767+0000", 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036768740000000001 2024-09-13T14:44:30.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.701146+0000", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037860299999999998 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.704384+0000", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032383099999999999 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.704471+0000", 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7294999999999997e-05 2024-09-13T14:44:30.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.704541+0000", 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9536999999999995e-05 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113885 8.e 8:78a75efd:::benchmark_data_smithi017_253885_object22544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.925403+0000", 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.209449575000001, 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074361239999999997, 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.240 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113885 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925403+0000", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925403+0000", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925405+0000", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6190000000000001e-06 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925412+0000", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.17e-06 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925432+0000", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9851999999999999e-05 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925443+0000", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1080000000000001e-05 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925500+0000", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6650000000000001e-05 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925601+0000", 2024-09-13T14:44:30.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101296 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.925700+0000", 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9178000000000005e-05 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.932653+0000", 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069525400000000001 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.932674+0000", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1250000000000002e-05 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.932726+0000", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1455000000000001e-05 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.932787+0000", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1764e-05 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.932840+0000", 2024-09-13T14:44:30.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2269999999999999e-05 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113934 8.c 8:3662794c:::benchmark_data_smithi017_253885_object22593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:09.634784+0000", 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 20.500068745, 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074571339999999998, 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113934 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634784+0000", 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634784+0000", 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634786+0000", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.815e-06 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634794+0000", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6499999999999996e-06 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634816+0000", 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2682999999999999e-05 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634826+0000", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4690000000000003e-06 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.634912+0000", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5983000000000007e-05 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.635046+0000", 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134306 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.635155+0000", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108539 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.640493+0000", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053385899999999998 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.641256+0000", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00076286300000000002 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.642158+0000", 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090154500000000002 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.642193+0000", 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5200000000000002e-05 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.642241+0000", 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8491e-05 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113940 8.c 8:3072b0a1:::benchmark_data_smithi017_253885_object22599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:09.646743+0000", 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 20.488109723000001, 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076129839999999997, 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113940 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:30.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646743+0000", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646743+0000", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646744+0000", 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0299999999999997e-07 2024-09-13T14:44:30.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646749+0000", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1109999999999999e-06 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646762+0000", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.314e-05 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646771+0000", 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2169999999999994e-06 2024-09-13T14:44:30.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646829+0000", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7946999999999999e-05 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.646953+0000", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124394 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.647058+0000", 2024-09-13T14:44:30.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105024 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.653550+0000", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064916280000000002 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.653598+0000", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8001999999999999e-05 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.654287+0000", 2024-09-13T14:44:30.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00068925099999999999 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.654324+0000", 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6869000000000002e-05 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:09.654356+0000", 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2498e-05 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:30.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:30.258 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:44:30.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:30.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:30.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:30.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:30 smithi189 ceph-mon[90992]: pgmap v1057: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38366/199275 objects degraded (19.253%); 4096/199275 objects misplaced (2.055%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:30.407 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:30.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:30 smithi017 ceph-mon[103001]: pgmap v1057: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38366/199275 objects degraded (19.253%); 4096/199275 objects misplaced (2.055%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:30.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:30 smithi017 ceph-mon[107556]: pgmap v1057: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 38366/199275 objects degraded (19.253%); 4096/199275 objects misplaced (2.055%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:30.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:30.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:30.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:30.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:30.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:30.751 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:30.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:30.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:30.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:30.904 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:31.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:31.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:31.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:31.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:31.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:31.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:31.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:31.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:31 smithi017 ceph-mon[103001]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:31 smithi017 ceph-mon[107556]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:31.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:31 smithi189 ceph-mon[90992]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:31.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:31.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:31.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:31.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:44:31.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:31.659 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:31.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:31.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:31.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:31.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:32.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:32.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:32.142 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:32.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:32 smithi017 ceph-mon[107556]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38366/198357 objects degraded (19.342%); 4096/198357 objects misplaced (2.065%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:32.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:32 smithi017 ceph-mon[103001]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38366/198357 objects degraded (19.342%); 4096/198357 objects misplaced (2.065%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:32.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:32.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:32.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:32.400 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1057,"stamp":"2024-09-13T14:44:29.207419+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4356016822,"num_objects":66425,"num_object_clones":0,"num_object_copies":199275,"num_objects_missing_on_primary":3144,"num_objects_missing":3144,"num_objects_degraded":38366,"num_objects_misplaced":4096,"num_objects_unfound":0,"num_objects_dirty":66425,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":117228,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22932,"num_bytes_recovered":1438877853,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117713,"ondisk_log_size":117713,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16703984,"kb_used_data":11719984,"kb_used_omap":44,"kb_used_meta":4983891,"kb_avail":733257232,"statfs":{"total":767960285184,"available":750855405568,"internally_reserved":0,"allocated":12001263616,"data_stored":11986876813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5103504631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,1,0,0,0,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96468992,"num_objects":-1472,"num_object_clones":0,"num_object_copies":-4416,"num_objects_missing_on_primary":-76,"num_objects_missing":-76,"num_objects_degraded":-250,"num_objects_misplaced":-78,"num_objects_unfound":0,"num_objects_dirty":-1472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1472,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":10813440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002901"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":577,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.279040+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:44:25.279040+0000","last_peered":"2024-09-13T14:44:25.279040+0000","last_clean":"2024-09-13T14:44:25.279040+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:44:25.279040+0000","last_undegraded":"2024-09-13T14:44:25.279040+0000","last_fullsized":"2024-09-13T14:44:25.279040+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":345,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479298+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:44:10.479298+0000","last_peered":"2024-09-13T14:44:10.479298+0000","last_clean":"2024-09-13T14:44:10.479298+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:44:10.479298+0000","last_undegraded":"2024-09-13T14:44:10.479298+0000","last_fullsized":"2024-09-13T14:44:10.479298+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":274,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128442+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:44:10.128442+0000","last_peered":"2024-09-13T14:44:10.128442+0000","last_clean":"2024-09-13T14:44:10.128442+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:44:10.128442+0000","last_undegraded":"2024-09-13T14:44:10.128442+0000","last_fullsized":"2024-09-13T14:44:10.128442+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":562,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344247+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:44:25.344247+0000","last_peered":"2024-09-13T14:44:25.344247+0000","last_clean":"2024-09-13T14:44:25.344247+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:44:25.344247+0000","last_undegraded":"2024-09-13T14:44:25.344247+0000","last_fullsized":"2024-09-13T14:44:25.344247+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":563,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344605+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:44:25.344605+0000","last_peered":"2024-09-13T14:44:25.344605+0000","last_clean":"2024-09-13T14:44:25.344605+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:44:25.344605+0000","last_undegraded":"2024-09-13T14:44:25.344605+0000","last_fullsized":"2024-09-13T14:44:25.344605+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":314,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479294+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:44:10.479294+0000","last_peered":"2024-09-13T14:44:10.479294+0000","last_clean":"2024-09-13T14:44:10.479294+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:44:10.479294+0000","last_undegraded":"2024-09-13T14:44:10.479294+0000","last_fullsized":"2024-09-13T14:44:10.479294+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":577,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.279399+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:44:25.279399+0000","last_peered":"2024-09-13T14:44:25.279399+0000","last_clean":"2024-09-13T14:44:25.279399+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:44:25.279399+0000","last_undegraded":"2024-09-13T14:44:25.279399+0000","last_fullsized":"2024-09-13T14:44:25.279399+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":336,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.475221+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:44:10.475221+0000","last_peered":"2024-09-13T14:44:10.475221+0000","last_clean":"2024-09-13T14:44:10.475221+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:44:10.475221+0000","last_undegraded":"2024-09-13T14:44:10.475221+0000","last_fullsized":"2024-09-13T14:44:10.475221+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"347'7216","reported_seq":12023,"reported_epoch":347,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:26.904092+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:44:26.904092+0000","last_peered":"2024-09-13T14:44:26.904092+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:44:26.904092+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7216,"log_dups_size":0,"ondisk_log_size":7216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266993664,"num_objects":4074,"num_object_clones":0,"num_object_copies":12222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7216,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1571,"num_bytes_recovered":102957056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":3090}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":340,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479374+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:44:10.479374+0000","last_peered":"2024-09-13T14:44:10.479374+0000","last_clean":"2024-09-13T14:44:10.479374+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:44:10.479374+0000","last_undegraded":"2024-09-13T14:44:10.479374+0000","last_fullsized":"2024-09-13T14:44:10.479374+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.127982+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:44:10.127982+0000","last_peered":"2024-09-13T14:44:10.127982+0000","last_clean":"2024-09-13T14:44:10.127982+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:44:10.127982+0000","last_undegraded":"2024-09-13T14:44:10.127982+0000","last_fullsized":"2024-09-13T14:44:10.127982+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"347'7312","reported_seq":16856,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:27.870262+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:44:27.870262+0000","last_peered":"2024-09-13T14:44:27.870262+0000","last_clean":"2024-09-13T14:44:27.870262+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:44:27.870262+0000","last_undegraded":"2024-09-13T14:44:27.870262+0000","last_fullsized":"2024-09-13T14:44:27.870262+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7312,"log_dups_size":0,"ondisk_log_size":7312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273678336,"num_objects":4176,"num_object_clones":0,"num_object_copies":12528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7312,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":358,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479556+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:44:10.479556+0000","last_peered":"2024-09-13T14:44:10.479556+0000","last_clean":"2024-09-13T14:44:10.479556+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:44:10.479556+0000","last_undegraded":"2024-09-13T14:44:10.479556+0000","last_fullsized":"2024-09-13T14:44:10.479556+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.134924+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:44:10.134924+0000","last_peered":"2024-09-13T14:44:10.134924+0000","last_clean":"2024-09-13T14:44:10.134924+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:44:10.134924+0000","last_undegraded":"2024-09-13T14:44:10.134924+0000","last_fullsized":"2024-09-13T14:44:10.134924+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"347'7556","reported_seq":7657,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:28.867068+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:44:28.867068+0000","last_peered":"2024-09-13T14:44:28.867068+0000","last_clean":"2024-09-13T14:44:28.867068+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:44:28.867068+0000","last_undegraded":"2024-09-13T14:44:28.867068+0000","last_fullsized":"2024-09-13T14:44:28.867068+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003712,"num_objects":4242,"num_object_clones":0,"num_object_copies":12726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7556,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":312,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.480243+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:44:10.480243+0000","last_peered":"2024-09-13T14:44:10.480243+0000","last_clean":"2024-09-13T14:44:10.480243+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:44:10.480243+0000","last_undegraded":"2024-09-13T14:44:10.480243+0000","last_fullsized":"2024-09-13T14:44:10.480243+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":327,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.475391+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:44:10.475391+0000","last_peered":"2024-09-13T14:44:10.475391+0000","last_clean":"2024-09-13T14:44:10.475391+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:44:10.475391+0000","last_undegraded":"2024-09-13T14:44:10.475391+0000","last_fullsized":"2024-09-13T14:44:10.475391+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"346'7351","reported_seq":7450,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:24.332685+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:44:24.332685+0000","last_peered":"2024-09-13T14:44:24.332685+0000","last_clean":"2024-09-13T14:44:24.332685+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:44:24.332685+0000","last_undegraded":"2024-09-13T14:44:24.332685+0000","last_fullsized":"2024-09-13T14:44:24.332685+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274923520,"num_objects":4195,"num_object_clones":0,"num_object_copies":12585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7351,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":275,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128290+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:44:10.128290+0000","last_peered":"2024-09-13T14:44:10.128290+0000","last_clean":"2024-09-13T14:44:10.128290+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:44:10.128290+0000","last_undegraded":"2024-09-13T14:44:10.128290+0000","last_fullsized":"2024-09-13T14:44:10.128290+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":589,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344580+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:44:25.344580+0000","last_peered":"2024-09-13T14:44:25.344580+0000","last_clean":"2024-09-13T14:44:25.344580+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:44:25.344580+0000","last_undegraded":"2024-09-13T14:44:25.344580+0000","last_fullsized":"2024-09-13T14:44:25.344580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"346'7270","reported_seq":7368,"reported_epoch":347,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:25.276435+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:44:25.276435+0000","last_peered":"2024-09-13T14:44:25.276435+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:44:25.276435+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274726912,"num_objects":4192,"num_object_clones":0,"num_object_copies":12576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7270,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1441}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":458,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479769+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:44:10.479769+0000","last_peered":"2024-09-13T14:44:10.479769+0000","last_clean":"2024-09-13T14:44:10.479769+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:44:10.479769+0000","last_undegraded":"2024-09-13T14:44:10.479769+0000","last_fullsized":"2024-09-13T14:44:10.479769+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":587,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.278679+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:44:25.278679+0000","last_peered":"2024-09-13T14:44:25.278679+0000","last_clean":"2024-09-13T14:44:25.278679+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:44:25.278679+0000","last_undegraded":"2024-09-13T14:44:25.278679+0000","last_fullsized":"2024-09-13T14:44:25.278679+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.138428+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:44:10.138428+0000","last_peered":"2024-09-13T14:44:10.138428+0000","last_clean":"2024-09-13T14:44:10.138428+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:44:10.138428+0000","last_undegraded":"2024-09-13T14:44:10.138428+0000","last_fullsized":"2024-09-13T14:44:10.138428+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"346'7003","reported_seq":7102,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.034682+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:44:25.034682+0000","last_peered":"2024-09-13T14:44:25.034682+0000","last_clean":"2024-09-13T14:44:25.034682+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:44:25.034682+0000","last_undegraded":"2024-09-13T14:44:25.034682+0000","last_fullsized":"2024-09-13T14:44:25.034682+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268107776,"num_objects":4091,"num_object_clones":0,"num_object_copies":12273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7003,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":162,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128062+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:44:10.128062+0000","last_peered":"2024-09-13T14:44:10.128062+0000","last_clean":"2024-09-13T14:44:10.128062+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:44:10.128062+0000","last_undegraded":"2024-09-13T14:44:10.128062+0000","last_fullsized":"2024-09-13T14:44:10.128062+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":348,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.474875+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:44:10.474875+0000","last_peered":"2024-09-13T14:44:10.474875+0000","last_clean":"2024-09-13T14:44:10.474875+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:44:10.474875+0000","last_undegraded":"2024-09-13T14:44:10.474875+0000","last_fullsized":"2024-09-13T14:44:10.474875+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":60,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135308+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:44:10.135308+0000","last_peered":"2024-09-13T14:44:10.135308+0000","last_clean":"2024-09-13T14:44:10.135308+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:44:10.135308+0000","last_undegraded":"2024-09-13T14:44:10.135308+0000","last_fullsized":"2024-09-13T14:44:10.135308+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"347'7213","reported_seq":11442,"reported_epoch":347,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:28.937489+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:44:28.937489+0000","last_peered":"2024-09-13T14:44:28.937489+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:44:28.937489+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7213,"log_dups_size":0,"ondisk_log_size":7213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275185664,"num_objects":4199,"num_object_clones":0,"num_object_copies":12597,"num_objects_missing_on_primary":3144,"num_objects_missing":3144,"num_objects_degraded":3519,"num_objects_misplaced":4096,"num_objects_unfound":0,"num_objects_dirty":4199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7213,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1507,"num_bytes_recovered":98762752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3144},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.134727+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:44:10.134727+0000","last_peered":"2024-09-13T14:44:10.134727+0000","last_clean":"2024-09-13T14:44:10.134727+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:44:10.134727+0000","last_undegraded":"2024-09-13T14:44:10.134727+0000","last_fullsized":"2024-09-13T14:44:10.134727+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135423+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:44:10.135423+0000","last_peered":"2024-09-13T14:44:10.135423+0000","last_clean":"2024-09-13T14:44:10.135423+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:44:10.135423+0000","last_undegraded":"2024-09-13T14:44:10.135423+0000","last_fullsized":"2024-09-13T14:44:10.135423+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"346'7083","reported_seq":7182,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:24.025853+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:44:24.025853+0000","last_peered":"2024-09-13T14:44:24.025853+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:44:24.025853+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7083,"log_dups_size":0,"ondisk_log_size":7083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267059200,"num_objects":4075,"num_object_clones":0,"num_object_copies":12225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7083,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1391}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":155,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128286+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:44:10.128286+0000","last_peered":"2024-09-13T14:44:10.128286+0000","last_clean":"2024-09-13T14:44:10.128286+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:44:10.128286+0000","last_undegraded":"2024-09-13T14:44:10.128286+0000","last_fullsized":"2024-09-13T14:44:10.128286+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":579,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.285264+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:44:25.285264+0000","last_peered":"2024-09-13T14:44:25.285264+0000","last_clean":"2024-09-13T14:44:25.285264+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:44:25.285264+0000","last_undegraded":"2024-09-13T14:44:25.285264+0000","last_fullsized":"2024-09-13T14:44:25.285264+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":539,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129182+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:44:10.129182+0000","last_peered":"2024-09-13T14:44:10.129182+0000","last_clean":"2024-09-13T14:44:10.129182+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:44:10.129182+0000","last_undegraded":"2024-09-13T14:44:10.129182+0000","last_fullsized":"2024-09-13T14:44:10.129182+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"347'7160","reported_seq":7219,"reported_epoch":347,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:27.834222+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:44:27.834222+0000","last_peered":"2024-09-13T14:44:27.834222+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:44:27.834222+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7160,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1436}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":449,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.482151+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:44:10.482151+0000","last_peered":"2024-09-13T14:44:10.482151+0000","last_clean":"2024-09-13T14:44:10.482151+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:44:10.482151+0000","last_undegraded":"2024-09-13T14:44:10.482151+0000","last_fullsized":"2024-09-13T14:44:10.482151+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":871,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128187+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:44:10.128187+0000","last_peered":"2024-09-13T14:44:10.128187+0000","last_clean":"2024-09-13T14:44:10.128187+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:44:10.128187+0000","last_undegraded":"2024-09-13T14:44:10.128187+0000","last_fullsized":"2024-09-13T14:44:10.128187+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":532,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.277381+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:44:25.277381+0000","last_peered":"2024-09-13T14:44:25.277381+0000","last_clean":"2024-09-13T14:44:25.277381+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:44:25.277381+0000","last_undegraded":"2024-09-13T14:44:25.277381+0000","last_fullsized":"2024-09-13T14:44:25.277381+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":313,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.474900+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:44:10.474900+0000","last_peered":"2024-09-13T14:44:10.474900+0000","last_clean":"2024-09-13T14:44:10.474900+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:44:10.474900+0000","last_undegraded":"2024-09-13T14:44:10.474900+0000","last_fullsized":"2024-09-13T14:44:10.474900+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"346'7346","reported_seq":7443,"reported_epoch":346,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:23.919736+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:44:23.919736+0000","last_peered":"2024-09-13T14:44:23.919736+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:44:23.919736+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7346,"log_dups_size":0,"ondisk_log_size":7346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275382272,"num_objects":4202,"num_object_clones":0,"num_object_copies":12606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7346,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1478}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":384,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.131651+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:44:10.131651+0000","last_peered":"2024-09-13T14:44:10.131651+0000","last_clean":"2024-09-13T14:44:10.131651+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:44:10.131651+0000","last_undegraded":"2024-09-13T14:44:10.131651+0000","last_fullsized":"2024-09-13T14:44:10.131651+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":737,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.482123+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:44:10.482123+0000","last_peered":"2024-09-13T14:44:10.482123+0000","last_clean":"2024-09-13T14:44:10.482123+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:44:10.482123+0000","last_undegraded":"2024-09-13T14:44:10.482123+0000","last_fullsized":"2024-09-13T14:44:10.482123+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":581,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344192+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:44:25.344192+0000","last_peered":"2024-09-13T14:44:25.344192+0000","last_clean":"2024-09-13T14:44:25.344192+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:44:25.344192+0000","last_undegraded":"2024-09-13T14:44:25.344192+0000","last_fullsized":"2024-09-13T14:44:25.344192+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":586,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344667+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:44:25.344667+0000","last_peered":"2024-09-13T14:44:25.344667+0000","last_clean":"2024-09-13T14:44:25.344667+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:44:25.344667+0000","last_undegraded":"2024-09-13T14:44:25.344667+0000","last_fullsized":"2024-09-13T14:44:25.344667+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"347'7369","reported_seq":17011,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.574034+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:44:25.574034+0000","last_peered":"2024-09-13T14:44:25.574034+0000","last_clean":"2024-09-13T14:44:25.574034+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:44:25.574034+0000","last_undegraded":"2024-09-13T14:44:25.574034+0000","last_fullsized":"2024-09-13T14:44:25.574034+0000","mapping_epoch":330,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281477120,"num_objects":4295,"num_object_clones":0,"num_object_copies":12885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7369,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":478,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479302+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:44:10.479302+0000","last_peered":"2024-09-13T14:44:10.479302+0000","last_clean":"2024-09-13T14:44:10.479302+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:44:10.479302+0000","last_undegraded":"2024-09-13T14:44:10.479302+0000","last_fullsized":"2024-09-13T14:44:10.479302+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":550,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.277163+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:44:25.277163+0000","last_peered":"2024-09-13T14:44:25.277163+0000","last_clean":"2024-09-13T14:44:25.277163+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:44:25.277163+0000","last_undegraded":"2024-09-13T14:44:25.277163+0000","last_fullsized":"2024-09-13T14:44:25.277163+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":516,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128977+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:44:10.128977+0000","last_peered":"2024-09-13T14:44:10.128977+0000","last_clean":"2024-09-13T14:44:10.128977+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:44:10.128977+0000","last_undegraded":"2024-09-13T14:44:10.128977+0000","last_fullsized":"2024-09-13T14:44:10.128977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":96,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128103+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:44:10.128103+0000","last_peered":"2024-09-13T14:44:10.128103+0000","last_clean":"2024-09-13T14:44:10.128103+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:44:10.128103+0000","last_undegraded":"2024-09-13T14:44:10.128103+0000","last_fullsized":"2024-09-13T14:44:10.128103+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"347'7095","reported_seq":7195,"reported_epoch":347,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:25.282732+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:44:25.282732+0000","last_peered":"2024-09-13T14:44:25.282732+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:44:25.282732+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7095,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1374}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":155,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128032+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:44:10.128032+0000","last_peered":"2024-09-13T14:44:10.128032+0000","last_clean":"2024-09-13T14:44:10.128032+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:44:10.128032+0000","last_undegraded":"2024-09-13T14:44:10.128032+0000","last_fullsized":"2024-09-13T14:44:10.128032+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":962,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128361+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:44:10.128361+0000","last_peered":"2024-09-13T14:44:10.128361+0000","last_clean":"2024-09-13T14:44:10.128361+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:44:10.128361+0000","last_undegraded":"2024-09-13T14:44:10.128361+0000","last_fullsized":"2024-09-13T14:44:10.128361+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135209+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:44:10.135209+0000","last_peered":"2024-09-13T14:44:10.135209+0000","last_clean":"2024-09-13T14:44:10.135209+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:44:10.135209+0000","last_undegraded":"2024-09-13T14:44:10.135209+0000","last_fullsized":"2024-09-13T14:44:10.135209+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":537,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129101+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:44:10.129101+0000","last_peered":"2024-09-13T14:44:10.129101+0000","last_clean":"2024-09-13T14:44:10.129101+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:44:10.129101+0000","last_undegraded":"2024-09-13T14:44:10.129101+0000","last_fullsized":"2024-09-13T14:44:10.129101+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"347'7268","reported_seq":16649,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:28.928654+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:44:28.928654+0000","last_peered":"2024-09-13T14:44:28.928654+0000","last_clean":"2024-09-13T14:44:28.928654+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:44:28.928654+0000","last_undegraded":"2024-09-13T14:44:28.928654+0000","last_fullsized":"2024-09-13T14:44:28.928654+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7268,"log_dups_size":0,"ondisk_log_size":7268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268173312,"num_objects":4092,"num_object_clones":0,"num_object_copies":12276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7268,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":346,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.477128+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:44:10.477128+0000","last_peered":"2024-09-13T14:44:10.477128+0000","last_clean":"2024-09-13T14:44:10.477128+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:44:10.477128+0000","last_undegraded":"2024-09-13T14:44:10.477128+0000","last_fullsized":"2024-09-13T14:44:10.477128+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":292,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128325+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:44:10.128325+0000","last_peered":"2024-09-13T14:44:10.128325+0000","last_clean":"2024-09-13T14:44:10.128325+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:44:10.128325+0000","last_undegraded":"2024-09-13T14:44:10.128325+0000","last_fullsized":"2024-09-13T14:44:10.128325+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1116,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129286+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:44:10.129286+0000","last_peered":"2024-09-13T14:44:10.129286+0000","last_clean":"2024-09-13T14:44:10.129286+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:44:10.129286+0000","last_undegraded":"2024-09-13T14:44:10.129286+0000","last_fullsized":"2024-09-13T14:44:10.129286+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":518,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130875+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:44:10.130875+0000","last_peered":"2024-09-13T14:44:10.130875+0000","last_clean":"2024-09-13T14:44:10.130875+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:44:10.130875+0000","last_undegraded":"2024-09-13T14:44:10.130875+0000","last_fullsized":"2024-09-13T14:44:10.130875+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"346'7128","reported_seq":7229,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.279413+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:44:25.279413+0000","last_peered":"2024-09-13T14:44:25.279413+0000","last_clean":"2024-09-13T14:44:25.279413+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:44:25.279413+0000","last_undegraded":"2024-09-13T14:44:25.279413+0000","last_fullsized":"2024-09-13T14:44:25.279413+0000","mapping_epoch":303,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267124736,"num_objects":4076,"num_object_clones":0,"num_object_copies":12228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7128,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":346,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.476036+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:44:10.476036+0000","last_peered":"2024-09-13T14:44:10.476036+0000","last_clean":"2024-09-13T14:44:10.476036+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:44:10.476036+0000","last_undegraded":"2024-09-13T14:44:10.476036+0000","last_fullsized":"2024-09-13T14:44:10.476036+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":527,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129194+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:44:10.129194+0000","last_peered":"2024-09-13T14:44:10.129194+0000","last_clean":"2024-09-13T14:44:10.129194+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:44:10.129194+0000","last_undegraded":"2024-09-13T14:44:10.129194+0000","last_fullsized":"2024-09-13T14:44:10.129194+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":741,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.480320+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:44:10.480320+0000","last_peered":"2024-09-13T14:44:10.480320+0000","last_clean":"2024-09-13T14:44:10.480320+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:44:10.480320+0000","last_undegraded":"2024-09-13T14:44:10.480320+0000","last_fullsized":"2024-09-13T14:44:10.480320+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":564,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344459+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:44:25.344459+0000","last_peered":"2024-09-13T14:44:25.344459+0000","last_clean":"2024-09-13T14:44:25.344459+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:44:25.344459+0000","last_undegraded":"2024-09-13T14:44:25.344459+0000","last_fullsized":"2024-09-13T14:44:25.344459+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"347'7509","reported_seq":7610,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:28.937165+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:44:28.937165+0000","last_peered":"2024-09-13T14:44:28.937165+0000","last_clean":"2024-09-13T14:44:28.937165+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:44:28.937165+0000","last_undegraded":"2024-09-13T14:44:28.937165+0000","last_fullsized":"2024-09-13T14:44:28.937165+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7509,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135205+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:44:10.135205+0000","last_peered":"2024-09-13T14:44:10.135205+0000","last_clean":"2024-09-13T14:44:10.135205+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:44:10.135205+0000","last_undegraded":"2024-09-13T14:44:10.135205+0000","last_fullsized":"2024-09-13T14:44:10.135205+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":565,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.279636+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:44:25.279636+0000","last_peered":"2024-09-13T14:44:25.279636+0000","last_clean":"2024-09-13T14:44:25.279636+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:44:25.279636+0000","last_undegraded":"2024-09-13T14:44:25.279636+0000","last_fullsized":"2024-09-13T14:44:25.279636+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":903,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129780+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:44:10.129780+0000","last_peered":"2024-09-13T14:44:10.129780+0000","last_clean":"2024-09-13T14:44:10.129780+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:44:10.129780+0000","last_undegraded":"2024-09-13T14:44:10.129780+0000","last_fullsized":"2024-09-13T14:44:10.129780+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":563,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.285212+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:44:25.285212+0000","last_peered":"2024-09-13T14:44:25.285212+0000","last_clean":"2024-09-13T14:44:25.285212+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:44:25.285212+0000","last_undegraded":"2024-09-13T14:44:25.285212+0000","last_fullsized":"2024-09-13T14:44:25.285212+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"347'7402","reported_seq":18760,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:27.637839+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:44:27.637839+0000","last_peered":"2024-09-13T14:44:27.637839+0000","last_clean":"2024-09-13T14:44:27.637839+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:44:27.637839+0000","last_undegraded":"2024-09-13T14:44:27.637839+0000","last_fullsized":"2024-09-13T14:44:27.637839+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7402,"log_dups_size":0,"ondisk_log_size":7402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269418520,"num_objects":4112,"num_object_clones":0,"num_object_copies":12336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4112,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7402,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":416,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.278875+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:44:25.278875+0000","last_peered":"2024-09-13T14:44:25.278875+0000","last_clean":"2024-09-13T14:44:25.278875+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:44:25.278875+0000","last_undegraded":"2024-09-13T14:44:25.278875+0000","last_fullsized":"2024-09-13T14:44:25.278875+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":600,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344558+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:44:25.344558+0000","last_peered":"2024-09-13T14:44:25.344558+0000","last_clean":"2024-09-13T14:44:25.344558+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:44:25.344558+0000","last_undegraded":"2024-09-13T14:44:25.344558+0000","last_fullsized":"2024-09-13T14:44:25.344558+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":765,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.477199+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:44:10.477199+0000","last_peered":"2024-09-13T14:44:10.477199+0000","last_clean":"2024-09-13T14:44:10.477199+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:44:10.477199+0000","last_undegraded":"2024-09-13T14:44:10.477199+0000","last_fullsized":"2024-09-13T14:44:10.477199+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.134864+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:44:10.134864+0000","last_peered":"2024-09-13T14:44:10.134864+0000","last_clean":"2024-09-13T14:44:10.134864+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:44:10.134864+0000","last_undegraded":"2024-09-13T14:44:10.134864+0000","last_fullsized":"2024-09-13T14:44:10.134864+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135278+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:44:10.135278+0000","last_peered":"2024-09-13T14:44:10.135278+0000","last_clean":"2024-09-13T14:44:10.135278+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:44:10.135278+0000","last_undegraded":"2024-09-13T14:44:10.135278+0000","last_fullsized":"2024-09-13T14:44:10.135278+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":56,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135048+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:44:10.135048+0000","last_peered":"2024-09-13T14:44:10.135048+0000","last_clean":"2024-09-13T14:44:10.135048+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:44:10.135048+0000","last_undegraded":"2024-09-13T14:44:10.135048+0000","last_fullsized":"2024-09-13T14:44:10.135048+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":515,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130753+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:44:10.130753+0000","last_peered":"2024-09-13T14:44:10.130753+0000","last_clean":"2024-09-13T14:44:10.130753+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:44:10.130753+0000","last_undegraded":"2024-09-13T14:44:10.130753+0000","last_fullsized":"2024-09-13T14:44:10.130753+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":346,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.479425+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:44:10.479425+0000","last_peered":"2024-09-13T14:44:10.479425+0000","last_clean":"2024-09-13T14:44:10.479425+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:44:10.479425+0000","last_undegraded":"2024-09-13T14:44:10.479425+0000","last_fullsized":"2024-09-13T14:44:10.479425+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":312,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128169+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:44:10.128169+0000","last_peered":"2024-09-13T14:44:10.128169+0000","last_clean":"2024-09-13T14:44:10.128169+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:44:10.128169+0000","last_undegraded":"2024-09-13T14:44:10.128169+0000","last_fullsized":"2024-09-13T14:44:10.128169+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":521,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.276936+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:44:25.276936+0000","last_peered":"2024-09-13T14:44:25.276936+0000","last_clean":"2024-09-13T14:44:25.276936+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:44:25.276936+0000","last_undegraded":"2024-09-13T14:44:25.276936+0000","last_fullsized":"2024-09-13T14:44:25.276936+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":573,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344405+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:44:25.344405+0000","last_peered":"2024-09-13T14:44:25.344405+0000","last_clean":"2024-09-13T14:44:25.344405+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:44:25.344405+0000","last_undegraded":"2024-09-13T14:44:25.344405+0000","last_fullsized":"2024-09-13T14:44:25.344405+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":577,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.285158+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:44:25.285158+0000","last_peered":"2024-09-13T14:44:25.285158+0000","last_clean":"2024-09-13T14:44:25.285158+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:44:25.285158+0000","last_undegraded":"2024-09-13T14:44:25.285158+0000","last_fullsized":"2024-09-13T14:44:25.285158+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":344,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.477177+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:44:10.477177+0000","last_peered":"2024-09-13T14:44:10.477177+0000","last_clean":"2024-09-13T14:44:10.477177+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:44:10.477177+0000","last_undegraded":"2024-09-13T14:44:10.477177+0000","last_fullsized":"2024-09-13T14:44:10.477177+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":595,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.285269+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:44:25.285269+0000","last_peered":"2024-09-13T14:44:25.285269+0000","last_clean":"2024-09-13T14:44:25.285269+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:44:25.285269+0000","last_undegraded":"2024-09-13T14:44:25.285269+0000","last_fullsized":"2024-09-13T14:44:25.285269+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":565,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344616+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:44:25.344616+0000","last_peered":"2024-09-13T14:44:25.344616+0000","last_clean":"2024-09-13T14:44:25.344616+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:44:25.344616+0000","last_undegraded":"2024-09-13T14:44:25.344616+0000","last_fullsized":"2024-09-13T14:44:25.344616+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":540,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.130157+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:44:10.130157+0000","last_peered":"2024-09-13T14:44:10.130157+0000","last_clean":"2024-09-13T14:44:10.130157+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:44:10.130157+0000","last_undegraded":"2024-09-13T14:44:10.130157+0000","last_fullsized":"2024-09-13T14:44:10.130157+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":591,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.277762+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:44:25.277762+0000","last_peered":"2024-09-13T14:44:25.277762+0000","last_clean":"2024-09-13T14:44:25.277762+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:44:25.277762+0000","last_undegraded":"2024-09-13T14:44:25.277762+0000","last_fullsized":"2024-09-13T14:44:25.277762+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":45,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.145085+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:44:10.145085+0000","last_peered":"2024-09-13T14:44:10.145085+0000","last_clean":"2024-09-13T14:44:10.145085+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:44:10.145085+0000","last_undegraded":"2024-09-13T14:44:10.145085+0000","last_fullsized":"2024-09-13T14:44:10.145085+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":486,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.129450+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:44:10.129450+0000","last_peered":"2024-09-13T14:44:10.129450+0000","last_clean":"2024-09-13T14:44:10.129450+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:44:10.129450+0000","last_undegraded":"2024-09-13T14:44:10.129450+0000","last_fullsized":"2024-09-13T14:44:10.129450+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":569,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344345+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:44:25.344345+0000","last_peered":"2024-09-13T14:44:25.344345+0000","last_clean":"2024-09-13T14:44:25.344345+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:44:25.344345+0000","last_undegraded":"2024-09-13T14:44:25.344345+0000","last_fullsized":"2024-09-13T14:44:25.344345+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.135135+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:44:10.135135+0000","last_peered":"2024-09-13T14:44:10.135135+0000","last_clean":"2024-09-13T14:44:10.135135+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:44:10.135135+0000","last_undegraded":"2024-09-13T14:44:10.135135+0000","last_fullsized":"2024-09-13T14:44:10.135135+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":312,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128378+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:44:10.128378+0000","last_peered":"2024-09-13T14:44:10.128378+0000","last_clean":"2024-09-13T14:44:10.128378+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:44:10.128378+0000","last_undegraded":"2024-09-13T14:44:10.128378+0000","last_fullsized":"2024-09-13T14:44:10.128378+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.145001+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:44:10.145001+0000","last_peered":"2024-09-13T14:44:10.145001+0000","last_clean":"2024-09-13T14:44:10.145001+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:44:10.145001+0000","last_undegraded":"2024-09-13T14:44:10.145001+0000","last_fullsized":"2024-09-13T14:44:10.145001+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":278,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.128477+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:44:10.128477+0000","last_peered":"2024-09-13T14:44:10.128477+0000","last_clean":"2024-09-13T14:44:10.128477+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:44:10.128477+0000","last_undegraded":"2024-09-13T14:44:10.128477+0000","last_fullsized":"2024-09-13T14:44:10.128477+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":587,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.277238+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:44:25.277238+0000","last_peered":"2024-09-13T14:44:25.277238+0000","last_clean":"2024-09-13T14:44:25.277238+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:44:25.277238+0000","last_undegraded":"2024-09-13T14:44:25.277238+0000","last_fullsized":"2024-09-13T14:44:25.277238+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":574,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.279115+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:44:25.279115+0000","last_peered":"2024-09-13T14:44:25.279115+0000","last_clean":"2024-09-13T14:44:25.279115+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:44:25.279115+0000","last_undegraded":"2024-09-13T14:44:25.279115+0000","last_fullsized":"2024-09-13T14:44:25.279115+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.134672+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:44:10.134672+0000","last_peered":"2024-09-13T14:44:10.134672+0000","last_clean":"2024-09-13T14:44:10.134672+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:44:10.134672+0000","last_undegraded":"2024-09-13T14:44:10.134672+0000","last_fullsized":"2024-09-13T14:44:10.134672+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":578,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.344472+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:44:25.344472+0000","last_peered":"2024-09-13T14:44:25.344472+0000","last_clean":"2024-09-13T14:44:25.344472+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:44:25.344472+0000","last_undegraded":"2024-09-13T14:44:25.344472+0000","last_fullsized":"2024-09-13T14:44:25.344472+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4351328280,"num_objects":66397,"num_object_clones":0,"num_object_copies":199191,"num_objects_missing_on_primary":3144,"num_objects_missing":3144,"num_objects_degraded":38366,"num_objects_misplaced":4096,"num_objects_unfound":0,"num_objects_dirty":66397,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116281,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22875,"num_bytes_recovered":1438711808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11943030784,"data_stored":11943018568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115581,"ondisk_log_size":115581,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397900,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337516,"kb_used_data":1449828,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":91407636,"statfs":{"total":95995035648,"available":93601419264,"internally_reserved":0,"allocated":1484623872,"data_stored":1482808004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594128,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2284120,"kb_used_data":1685904,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":91461032,"statfs":{"total":95995035648,"available":93656096768,"internally_reserved":0,"allocated":1726365696,"data_stored":1724569582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665582,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643544,"kb_used_data":315152,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93101608,"statfs":{"total":95995035648,"available":95336046592,"internally_reserved":0,"allocated":322715648,"data_stored":320915865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986584,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1836132,"kb_used_data":936284,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":91909020,"statfs":{"total":95995035648,"available":94114836480,"internally_reserved":0,"allocated":958754816,"data_stored":956947046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,0,0,0,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697143,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2689540,"kb_used_data":1971644,"kb_used_omap":4,"kb_used_meta":717883,"kb_avail":91055612,"statfs":{"total":95995035648,"available":93240946688,"internally_reserved":0,"allocated":2018963456,"data_stored":2017175673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":735112267},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860670,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2509788,"kb_used_data":1888532,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":91235364,"statfs":{"total":95995035648,"available":93425012736,"internally_reserved":0,"allocated":1933856768,"data_stored":1932053484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020341,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181376,"kb_used_data":1873656,"kb_used_omap":9,"kb_used_meta":307702,"kb_avail":91563776,"statfs":{"total":95995035648,"available":93761306624,"internally_reserved":0,"allocated":1918623744,"data_stored":1916839434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":315087685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187732,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2221968,"kb_used_data":1598984,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":91523184,"statfs":{"total":95995035648,"available":93719740416,"internally_reserved":0,"allocated":1637359616,"data_stored":1635567725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1626144768,"data_stored":1626144768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1909522432,"data_stored":1909522432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1928335360,"data_stored":1928331288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2012413952,"data_stored":2012413952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":953548800,"data_stored":953548800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":317198336,"data_stored":317194264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1721040896,"data_stored":1721040896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1474826240,"data_stored":1474822168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:32.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:44:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:32.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:32 smithi189 ceph-mon[90992]: pgmap v1058: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 38366/198357 objects degraded (19.342%); 4096/198357 objects misplaced (2.065%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:44:32.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:32.539 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:32.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:32.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:32.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:32.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:32.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:32.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:32.980 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:32.981 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:33.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:33.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:33.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:33.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:33.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:33.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:33.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:33.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:33.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:33.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:33.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:33.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:33.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:33.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:33.775 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:33.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:33.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:33.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:34.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:34.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:34.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:34.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:34.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:34.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:34.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:34.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:34.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:34.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:34 smithi017 ceph-mon[103001]: pgmap v1059: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 38262/197841 objects degraded (19.340%); 4096/197841 objects misplaced (2.070%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:34.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38262/197841 objects degraded (19.340%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:34.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:34 smithi017 ceph-mon[107556]: pgmap v1059: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 38262/197841 objects degraded (19.340%); 4096/197841 objects misplaced (2.070%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:34.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38262/197841 objects degraded (19.340%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:34.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:34.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:34.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:34.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:34.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:34.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:34 smithi189 ceph-mon[90992]: pgmap v1059: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 38262/197841 objects degraded (19.340%); 4096/197841 objects misplaced (2.070%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38262/197841 objects degraded (19.340%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:34.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:34.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:34.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:35.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:35.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:35.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:35.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:35.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:35.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:35.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:35.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:35.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:35.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:35.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:35.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:35.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:35.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:35.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:35.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:35.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:35.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:36.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:36.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:36.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:36.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:36.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:36.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:36.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:36.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:36.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:36 smithi017 ceph-mon[103001]: pgmap v1060: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 38221/196929 objects degraded (19.409%); 4056/196929 objects misplaced (2.060%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:36.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:36 smithi017 ceph-mon[107556]: pgmap v1060: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 38221/196929 objects degraded (19.409%); 4056/196929 objects misplaced (2.060%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:36.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:36.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:36.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:36.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:36 smithi189 ceph-mon[90992]: pgmap v1060: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 38221/196929 objects degraded (19.409%); 4056/196929 objects misplaced (2.060%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:36.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:36.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:36.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:36.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:36.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:37.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:37.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:37.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:37.269 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:44:37.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:37.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:37.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:37.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:37.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:37.601 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:37] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:44:37.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:37.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:37.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:38.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:38.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:38.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:38.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:38.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:38.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[107556]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 38147/196380 objects degraded (19.425%); 4056/196380 objects misplaced (2.065%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[107556]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[103001]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 38147/196380 objects degraded (19.425%); 4056/196380 objects misplaced (2.065%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:38 smithi017 ceph-mon[103001]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:44:38.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:38.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:38.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:38.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:38.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:38.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:38 smithi189 ceph-mon[90992]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 38147/196380 objects degraded (19.425%); 4056/196380 objects misplaced (2.065%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:44:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:38 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/186417432' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:38 smithi189 ceph-mon[90992]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:44:38.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:38.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:38.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:38.851 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:38.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:38.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:38.875 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:38.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:38.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:38.891 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:38.891 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:38.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:38.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:38.908 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:38.908 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:38.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:38.926 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:38.942 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:38.943 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:38.958 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:38.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:39.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:39.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:39.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:39.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:39.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:39.280 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:39.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:39.407 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:39.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38147/196380 objects degraded (19.425%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4206809793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38147/196380 objects degraded (19.425%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4206809793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:39.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:39.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:39.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:39.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38147/196380 objects degraded (19.425%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4206809793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:39.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:39.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:39.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:39.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:40.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:40.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:40.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:40.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:40.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:40.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:40.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:40.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:40.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:40 smithi017 ceph-mon[103001]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 38106/195360 objects degraded (19.506%); 4016/195360 objects misplaced (2.056%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:40 smithi017 ceph-mon[107556]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 38106/195360 objects degraded (19.506%); 4016/195360 objects misplaced (2.056%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:40 smithi189 ceph-mon[90992]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 38106/195360 objects degraded (19.506%); 4016/195360 objects misplaced (2.056%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:40.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:40.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:44:40.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:40.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:40.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:40.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:40.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:40.866 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:40.866 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:40.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:40.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:41.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:41.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:41.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:41.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:41.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:41.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:41.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:41.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:41.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:41.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:41.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:41.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:41.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:44:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:41.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:41.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:42.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:42.067 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:42.073 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:42.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:42.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:42.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:42.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:42.302 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 38106/194370 objects degraded (19.605%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1479,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":348,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":64790,"data_bytes":4248865462,"bytes_used":16864919552,"bytes_avail":751095365632,"bytes_total":767960285184,"degraded_objects":38106,"degraded_total":194370,"degraded_ratio":0.19604877295878997,"misplaced_objects":4016,"misplaced_total":194370,"misplaced_ratio":0.02066162473632762,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1107218,"recovering_keys_per_sec":0,"num_objects_recovered":169,"num_bytes_recovered":11075584,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":132},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (5m)\n [==========================..] (remaining: 25s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:44:42.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:42.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:42.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[103001]: pgmap v1064: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38106/194370 objects degraded (19.605%); 4016/194370 objects misplaced (2.066%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:42.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:42.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:42.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[107556]: pgmap v1064: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38106/194370 objects degraded (19.605%); 4016/194370 objects misplaced (2.066%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:42.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:42.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:42.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:42.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:42.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:42.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:42.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:42.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:42 smithi189 ceph-mon[90992]: pgmap v1064: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 38106/194370 objects degraded (19.605%); 4016/194370 objects misplaced (2.066%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:44:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:44:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:42.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:42.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:42.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:42.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:43.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:43.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:43.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:43.214 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:43.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:43.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:43.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:43.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:43.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:43.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3148028692' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:43.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3148028692' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:43.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:43.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3148028692' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:44:43.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:43.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:43.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:44.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:44.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:44.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:44.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:44.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:44.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:44.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:44.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:44.526 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:44:44.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:44.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:44 smithi017 ceph-mon[103001]: pgmap v1065: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 38038/194010 objects degraded (19.606%); 4016/194010 objects misplaced (2.070%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:44:44.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 38038/194010 objects degraded (19.606%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:44.549 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:44 smithi017 ceph-mon[107556]: pgmap v1065: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 38038/194010 objects degraded (19.606%); 4016/194010 objects misplaced (2.070%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:44:44.549 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 38038/194010 objects degraded (19.606%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:44.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:44.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:44.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:44.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:44 smithi189 ceph-mon[90992]: pgmap v1065: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 38038/194010 objects degraded (19.606%); 4016/194010 objects misplaced (2.070%); 966 KiB/s, 15 objects/s recovering 2024-09-13T14:44:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 38038/194010 objects degraded (19.606%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:44.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:44.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:44.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:45.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:45.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:45.186 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:45.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:45.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:45.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:45.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:45.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:45.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:45.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:45.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:45.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:45.758 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:45.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:45.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:45.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:46.012 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:46.012 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:46.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:46.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:46.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:46.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:46.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:46.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:46 smithi017 ceph-mon[103001]: pgmap v1066: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37997/193173 objects degraded (19.670%); 3973/193173 objects misplaced (2.057%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:46.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:46 smithi017 ceph-mon[107556]: pgmap v1066: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37997/193173 objects degraded (19.670%); 3973/193173 objects misplaced (2.057%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:46.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:46.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:46.696 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:46.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:46.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:46 smithi189 ceph-mon[90992]: pgmap v1066: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37997/193173 objects degraded (19.670%); 3973/193173 objects misplaced (2.057%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:46.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:46.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:46.946 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:46.968 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:47.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:47.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:47.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:44:47.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:47.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:47.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:47.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:47] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:44:47.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:47.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:47.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:47.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:47.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:47.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:47.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:47.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:48.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:48.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:48.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:48.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:48.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:48.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:48.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:48 smithi017 ceph-mon[103001]: pgmap v1067: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37931/192711 objects degraded (19.683%); 3973/192711 objects misplaced (2.062%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:48 smithi017 ceph-mon[107556]: pgmap v1067: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37931/192711 objects degraded (19.683%); 3973/192711 objects misplaced (2.062%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:48.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:48.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:48.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:48.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:48.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:48.724 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:48 smithi189 ceph-mon[90992]: pgmap v1067: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37931/192711 objects degraded (19.683%); 3973/192711 objects misplaced (2.062%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:48.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:48.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:48.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:48.902 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:48.902 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:48.912 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:48.915 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:48.915 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:48.920 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:48.935 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:48.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:48.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:48.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:48.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:48.974 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:48.974 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:48.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:48.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:48.990 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:49.006 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:49.006 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:49.021 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:49.021 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:49.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:49.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:49.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.288 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37931/192711 objects degraded (19.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1552413829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37931/192711 objects degraded (19.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:49.307 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1552413829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.330 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:49.351 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:49.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:49.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:49.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:49.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:49.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:49.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:49.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37931/192711 objects degraded (19.683%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1552413829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:49.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:49.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:49.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:49.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:50.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:50.096 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:50.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:50.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:50.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:50.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:50.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:50.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:44:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:44:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:50.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:50 smithi189 ceph-mon[90992]: pgmap v1068: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 37888/191859 objects degraded (19.748%); 3933/191859 objects misplaced (2.050%); 1018 KiB/s, 15 objects/s recovering 2024-09-13T14:44:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:50 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:50 smithi189 ceph-mon[90992]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:44:50.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:50.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:50.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[103001]: pgmap v1068: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 37888/191859 objects degraded (19.748%); 3933/191859 objects misplaced (2.050%); 1018 KiB/s, 15 objects/s recovering 2024-09-13T14:44:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:50.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[103001]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:44:50.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[107556]: pgmap v1068: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 37888/191859 objects degraded (19.748%); 3933/191859 objects misplaced (2.050%); 1018 KiB/s, 15 objects/s recovering 2024-09-13T14:44:50.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:50.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1283937632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:50.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:50 smithi017 ceph-mon[107556]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:44:50.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:50.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:50.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:50.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:50.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:51.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:51.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:51.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:51.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:51.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:51.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:51.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:51.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:51.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:51.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:52.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:52.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:52.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:52.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:52.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:52.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:52.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:52.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:52.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:52.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:52.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:52.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:52 smithi017 ceph-mon[107556]: pgmap v1070: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 37888/190965 objects degraded (19.840%); 3933/190965 objects misplaced (2.060%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:52 smithi017 ceph-mon[103001]: pgmap v1070: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 37888/190965 objects degraded (19.840%); 3933/190965 objects misplaced (2.060%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:52.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:52.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:52 smithi189 ceph-mon[90992]: pgmap v1070: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 37888/190965 objects degraded (19.840%); 3933/190965 objects misplaced (2.060%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:44:52.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:44:52.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:52.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:52.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:53.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:53.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:53.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:53.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:53.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:53.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:53.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:53.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:53.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:53.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:53.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:53.819 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:44:53.821 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:44:53.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:53.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:53.954 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:44:53.954 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89680 8.4 8:21ec7e63:::benchmark_data_smithi017_253885_object89679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 250.23448624299999, 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3691795170000001, 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89680 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716046+0000", 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3740000000000001e-06 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716071+0000", 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4990999999999998e-05 2024-09-13T14:44:53.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716082+0000", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1008e-05 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716090+0000", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9459999999999998e-06 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716141+0000", 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0575999999999998e-05 2024-09-13T14:44:53.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716151+0000", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8810000000000008e-06 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699865+0000", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98371478300000004 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699883+0000", 2024-09-13T14:44:53.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8005000000000001e-05 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057337+0000", 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3574534739999999 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057492+0000", 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015483399999999999 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057629+0000", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137315 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064142+0000", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006512838 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.077471+0000", 2024-09-13T14:44:53.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013329314 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.081522+0000", 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040507049999999999 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.081591+0000", 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9319999999999994e-05 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:53.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085224+0000", 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036331530000000001 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89696 8.4 8:26ba4658:::benchmark_data_smithi017_253885_object89695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 250.209856476, 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3480544769999998, 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89696 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:44:53.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740676+0000", 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3039999999999999e-06 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740696+0000", 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9915000000000002e-05 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740708+0000", 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.222e-05 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740716+0000", 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.807e-06 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740785+0000", 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9041999999999994e-05 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740798+0000", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2714999999999999e-05 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699915+0000", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95911772799999995 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699925+0000", 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8910000000000007e-06 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064178+0000", 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.36425273 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064355+0000", 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177307 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:53.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064542+0000", 2024-09-13T14:44:53.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000186985 2024-09-13T14:44:53.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:53.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.067290+0000", 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00274727 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.077518+0000", 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010228331 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085295+0000", 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077774150000000002 2024-09-13T14:44:53.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085371+0000", 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5306e-05 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.088729+0000", 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033585109999999998 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:53.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89697 8.d 8:bb121d82:::benchmark_data_smithi017_253885_object89696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 249.25745370800001, 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.382211367, 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.972 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89697 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:44:53.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693080+0000", 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.362e-06 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693112+0000", 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2036999999999998e-05 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693137+0000", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4749999999999999e-05 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693148+0000", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1097999999999999e-05 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693757+0000", 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060958799999999999 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693765+0000", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2629999999999997e-06 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.697968+0000", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042034730000000001 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.697977+0000", 2024-09-13T14:44:53.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4450000000000002e-06 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.050416+0000", 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3524383870000001 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.053361+0000", 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029451220000000001 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.053511+0000", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000149949 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062676+0000", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091654820000000008 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062800+0000", 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012333600000000001 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.073014+0000", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010214076000000001 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.073062+0000", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8010999999999998e-05 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075289+0000", 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022269880000000001 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.979 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89699 8.d 8:b4b5c04d:::benchmark_data_smithi017_253885_object89698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 249.25735072000001, 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3850268240000001, 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.980 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89699 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693181+0000", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.99e-07 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693207+0000", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6259e-05 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693216+0000", 2024-09-13T14:44:53.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9760000000000001e-06 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693223+0000", 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9829999999999999e-06 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693816+0000", 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00059274000000000002 2024-09-13T14:44:53.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693821+0000", 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7010000000000004e-06 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.698001+0000", 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041799910000000001 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.698008+0000", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8900000000000001e-06 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059361+0000", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.361353665 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059552+0000", 2024-09-13T14:44:53.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019093800000000001 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059715+0000", 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016238900000000001 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062807+0000", 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030924160000000002 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075362+0000", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012554872 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075460+0000", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8325000000000005e-05 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075525+0000", 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4801000000000005e-05 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.078207+0000", 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026822790000000001 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:110827 8.4 8:2c2e0177:::benchmark_data_smithi017_253885_object19486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:43:43.693566+0000", 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.256965131000001, 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028351024999999998, 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.988 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 110827 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693566+0000", 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693566+0000", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693567+0000", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.367e-06 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693572+0000", 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8389999999999997e-06 2024-09-13T14:44:53.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693585+0000", 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2313999999999999e-05 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.693591+0000", 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.246e-06 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.697701+0000", 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041100859999999998 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.698193+0000", 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049226300000000005 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.712875+0000", 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014681493 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.712985+0000", 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110092 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.713133+0000", 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014810299999999999 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:53.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.717360+0000", 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042272799999999999 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.718565+0000", 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001204746 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.719294+0000", 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072864499999999999 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.719373+0000", 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9521e-05 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:43.721917+0000", 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025440300000000001 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:111020 8.4 8:22d34927:::benchmark_data_smithi017_253885_object19679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:43:45.287979+0000", 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 68.662552308000002, 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028853699999999999, 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 111020 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.996 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.287979+0000", 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.287979+0000", 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.287981+0000", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9999999999999999e-06 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.287987+0000", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3899999999999998e-06 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.288001+0000", 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3939e-05 2024-09-13T14:44:53.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.288008+0000", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7309999999999996e-06 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.295702+0000", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076936030000000002 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.296176+0000", 2024-09-13T14:44:53.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047483600000000002 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.307105+0000", 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010928251 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.307171+0000", 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6630000000000004e-05 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.307279+0000", 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107336 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.312296+0000", 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050170869999999999 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.314014+0000", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001718088 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.314194+0000", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000180426 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.314261+0000", 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6839000000000002e-05 2024-09-13T14:44:54.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:43:45.316833+0000", 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025715439999999998 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:112765 8.4 8:28f9c459:::benchmark_data_smithi017_253885_object21424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:02.095514+0000", 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 51.855017564999997, 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025923771000000002, 2024-09-13T14:44:54.003 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 112765 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095514+0000", 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095514+0000", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095515+0000", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.663e-06 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095522+0000", 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.216e-06 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095540+0000", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7694999999999999e-05 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095550+0000", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.995e-06 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.095612+0000", 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1661000000000005e-05 2024-09-13T14:44:54.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.096011+0000", 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000399279 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107642+0000", 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011630568000000001 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107712+0000", 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0155999999999997e-05 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107800+0000", 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8281000000000004e-05 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.111272+0000", 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034714989999999999 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.114066+0000", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027947979999999998 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.119364+0000", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052980040000000003 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.119435+0000", 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0356000000000002e-05 2024-09-13T14:44:54.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.121437+0000", 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020026000000000002 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:112769 8.4 8:241d5b68:::benchmark_data_smithi017_253885_object21428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:02.107180+0000", 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 51.843351028000001, 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025085006999999999, 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.010 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 112769 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107180+0000", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107180+0000", 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107181+0000", 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2529999999999999e-06 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107187+0000", 2024-09-13T14:44:54.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9320000000000001e-06 2024-09-13T14:44:54.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:54.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107200+0000", 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3016999999999999e-05 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.107208+0000", 2024-09-13T14:44:54.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6899999999999992e-06 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.110633+0000", 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034244710000000001 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.111249+0000", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00061658800000000005 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.124451+0000", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013201826 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.124526+0000", 2024-09-13T14:44:54.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4963999999999996e-05 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.124618+0000", 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1942999999999997e-05 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.129189+0000", 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045710100000000004 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.129585+0000", 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039630399999999999 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.129653+0000", 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8151999999999994e-05 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.129739+0000", 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5631999999999996e-05 2024-09-13T14:44:54.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.132265+0000", 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002526225 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:112861 8.4 8:23f97d1e:::benchmark_data_smithi017_253885_object21520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:02.933116+0000", 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 51.017414801000001, 2024-09-13T14:44:54.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025741107999999999, 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 112861 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933116+0000", 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933116+0000", 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933118+0000", 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.446e-06 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933122+0000", 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4689999999999999e-06 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933134+0000", 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1562999999999999e-05 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.933141+0000", 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6120000000000001e-06 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.935875+0000", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027332960000000001 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.936421+0000", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054637499999999999 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.946604+0000", 2024-09-13T14:44:54.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010182577 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.946677+0000", 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3656000000000001e-05 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.946770+0000", 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2776999999999996e-05 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.956573+0000", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098030949999999995 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.956594+0000", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0786000000000001e-05 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.956633+0000", 2024-09-13T14:44:54.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8566000000000003e-05 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.956686+0000", 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3300000000000001e-05 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.958857+0000", 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021715900000000002 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:112862 8.4 8:29a98547:::benchmark_data_smithi017_253885_object21521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:02.937040+0000", 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 51.013491449, 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024507688, 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 112862 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937040+0000", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937040+0000", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937041+0000", 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1000000000000001e-06 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937046+0000", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7199999999999997e-06 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937057+0000", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1371e-05 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937065+0000", 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1110000000000008e-06 2024-09-13T14:44:54.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937117+0000", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1576000000000002e-05 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.937505+0000", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038861100000000002 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.953467+0000", 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015961301000000001 2024-09-13T14:44:54.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.953545+0000", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8588e-05 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.953639+0000", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3490000000000001e-05 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.959653+0000", 2024-09-13T14:44:54.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060144409999999997 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.959687+0000", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4094000000000003e-05 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.959696+0000", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.602e-06 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.959727+0000", 2024-09-13T14:44:54.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1173999999999997e-05 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:02.961547+0000", 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018205090000000001 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113069 8.4 8:22f62c44:::benchmark_data_smithi017_253885_object21728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:04.079969+0000", 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 49.870562436, 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025417734000000001, 2024-09-13T14:44:54.034 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113069 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079969+0000", 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079969+0000", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079970+0000", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3629999999999999e-06 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079975+0000", 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1139999999999997e-06 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079988+0000", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2875e-05 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.079994+0000", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.105e-06 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.080059+0000", 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5067999999999994e-05 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.080626+0000", 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00056707000000000003 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.092961+0000", 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012334194999999999 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.093082+0000", 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121697 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.093252+0000", 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016975200000000001 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.103745+0000", 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010492669 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.103801+0000", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6257e-05 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.103888+0000", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6949999999999999e-05 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.103964+0000", 2024-09-13T14:44:54.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5840000000000006e-05 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.105386+0000", 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014227790000000001 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113071 8.4 8:2c831e42:::benchmark_data_smithi017_253885_object21730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:04.082334+0000", 2024-09-13T14:44:54.041 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 49.868197492999997, 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025906180000000001, 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113071 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082334+0000", 2024-09-13T14:44:54.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082334+0000", 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082335+0000", 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1969999999999999e-06 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082339+0000", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3059999999999999e-06 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082352+0000", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2763e-05 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082359+0000", 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5729999999999996e-06 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082417+0000", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8109000000000002e-05 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.082904+0000", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048770999999999998 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.100147+0000", 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017242568999999999 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.100265+0000", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011859 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.100424+0000", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015848300000000001 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.106129+0000", 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057052350000000003 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.106202+0000", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2502000000000005e-05 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.106221+0000", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9519999999999999e-05 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.106280+0000", 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9225000000000003e-05 2024-09-13T14:44:54.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.108240+0000", 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019593980000000002 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113073 8.4 8:2c350d1e:::benchmark_data_smithi017_253885_object21732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:04.088973+0000", 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 49.861558021999997, 2024-09-13T14:44:54.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041547489999999999, 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113073 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.088973+0000", 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.088973+0000", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.088975+0000", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.531e-06 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.088980+0000", 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6400000000000002e-06 2024-09-13T14:44:54.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.088993+0000", 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2447e-05 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.089000+0000", 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.041e-06 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.105438+0000", 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016438098000000002 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.106072+0000", 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00063425500000000002 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.114384+0000", 2024-09-13T14:44:54.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083121540000000004 2024-09-13T14:44:54.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.114497+0000", 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112959 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.114663+0000", 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016610799999999999 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.127555+0000", 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012891253 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.127575+0000", 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0339e-05 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.127637+0000", 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2261000000000006e-05 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.127716+0000", 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9149999999999999e-05 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.130521+0000", 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028042539999999999 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113074 8.4 8:21db2008:::benchmark_data_smithi017_253885_object21733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:04.104150+0000", 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 49.846380768000003, 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.030145295999999999, 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113074 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104150+0000", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104150+0000", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104152+0000", 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.336e-06 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104157+0000", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3519999999999997e-06 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104169+0000", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1637000000000001e-05 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.104174+0000", 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7309999999999998e-06 2024-09-13T14:44:54.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.108287+0000", 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041122620000000002 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.108871+0000", 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000583966 2024-09-13T14:44:54.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.123396+0000", 2024-09-13T14:44:54.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014525013 2024-09-13T14:44:54.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.123532+0000", 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013663099999999999 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.123721+0000", 2024-09-13T14:44:54.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018843700000000001 2024-09-13T14:44:54.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.131856+0000", 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081351040000000006 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.131874+0000", 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7968000000000001e-05 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.131911+0000", 2024-09-13T14:44:54.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7647000000000001e-05 2024-09-13T14:44:54.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.131969+0000", 2024-09-13T14:44:54.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7871999999999997e-05 2024-09-13T14:44:54.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:04.134296+0000", 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023263400000000001 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.070 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:113767 8.4 8:268febe5:::benchmark_data_smithi017_253885_object22426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.070 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:08.599349+0000", 2024-09-13T14:44:54.070 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 45.351182549000001, 2024-09-13T14:44:54.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028151392000000001, 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.071 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.072 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 113767 2024-09-13T14:44:54.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.072 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599349+0000", 2024-09-13T14:44:54.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599349+0000", 2024-09-13T14:44:54.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599350+0000", 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.705e-06 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599358+0000", 2024-09-13T14:44:54.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7219999999999999e-06 2024-09-13T14:44:54.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599375+0000", 2024-09-13T14:44:54.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6776e-05 2024-09-13T14:44:54.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.599384+0000", 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2420000000000006e-06 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.606095+0000", 2024-09-13T14:44:54.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067113729999999996 2024-09-13T14:44:54.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.606631+0000", 2024-09-13T14:44:54.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00053538499999999998 2024-09-13T14:44:54.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.618178+0000", 2024-09-13T14:44:54.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011546983 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.618272+0000", 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4388999999999994e-05 2024-09-13T14:44:54.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.618409+0000", 2024-09-13T14:44:54.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013655000000000001 2024-09-13T14:44:54.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.623332+0000", 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049232310000000001 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.624284+0000", 2024-09-13T14:44:54.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00095162700000000003 2024-09-13T14:44:54.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.624748+0000", 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046483100000000002 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.624816+0000", 2024-09-13T14:44:54.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7131000000000005e-05 2024-09-13T14:44:54.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:08.627500+0000", 2024-09-13T14:44:54.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002684447 2024-09-13T14:44:54.087 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.087 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.087 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.088 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:114029 8.4 8:2fb49c91:::benchmark_data_smithi017_253885_object22688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-09-13T14:44:54.088 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:10.119664+0000", 2024-09-13T14:44:54.088 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 43.830867634999997, 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025223782, 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.090 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.090 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 114029 2024-09-13T14:44:54.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.090 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119664+0000", 2024-09-13T14:44:54.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119664+0000", 2024-09-13T14:44:54.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119665+0000", 2024-09-13T14:44:54.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.232e-06 2024-09-13T14:44:54.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119670+0000", 2024-09-13T14:44:54.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.49e-06 2024-09-13T14:44:54.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119682+0000", 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1659e-05 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119688+0000", 2024-09-13T14:44:54.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6699999999999999e-06 2024-09-13T14:44:54.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.119740+0000", 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2784999999999997e-05 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.120124+0000", 2024-09-13T14:44:54.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038388700000000002 2024-09-13T14:44:54.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.133171+0000", 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013046958000000001 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.133284+0000", 2024-09-13T14:44:54.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112459 2024-09-13T14:44:54.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.133439+0000", 2024-09-13T14:44:54.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015499200000000001 2024-09-13T14:44:54.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.143650+0000", 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010211720000000001 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.143666+0000", 2024-09-13T14:44:54.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5746000000000001e-05 2024-09-13T14:44:54.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.143700+0000", 2024-09-13T14:44:54.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3556000000000002e-05 2024-09-13T14:44:54.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.143750+0000", 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0464000000000003e-05 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:10.144887+0000", 2024-09-13T14:44:54.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001137164 2024-09-13T14:44:54.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.106 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119269 8.4 8:2f9abc02:::benchmark_data_smithi017_253885_object27928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:51.672067+0000", 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.2784643290000002, 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028404572999999999, 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.107 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.108 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.108 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119269 2024-09-13T14:44:54.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.109 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672067+0000", 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672067+0000", 2024-09-13T14:44:54.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672069+0000", 2024-09-13T14:44:54.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0140000000000001e-06 2024-09-13T14:44:54.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672074+0000", 2024-09-13T14:44:54.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4480000000000002e-06 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672089+0000", 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4569999999999999e-05 2024-09-13T14:44:54.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672098+0000", 2024-09-13T14:44:54.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2e-06 2024-09-13T14:44:54.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672167+0000", 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8622999999999997e-05 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672623+0000", 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045680200000000002 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.683708+0000", 2024-09-13T14:44:54.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011084014999999999 2024-09-13T14:44:54.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.683822+0000", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011412 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.683990+0000", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016830999999999999 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.696365+0000", 2024-09-13T14:44:54.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012375247000000001 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.696415+0000", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9929999999999998e-05 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.697867+0000", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001451473 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.697927+0000", 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0426e-05 2024-09-13T14:44:54.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.700471+0000", 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025443950000000001 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119270 8.4 8:20636fc9:::benchmark_data_smithi017_253885_object27929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:51.672645+0000", 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.2778863619999998, 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.030565346, 2024-09-13T14:44:54.120 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119270 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672645+0000", 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672645+0000", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672646+0000", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0800000000000004e-07 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672650+0000", 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9029999999999997e-06 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672662+0000", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2611e-05 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672671+0000", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.901e-06 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.672719+0000", 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7849999999999998e-05 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.673136+0000", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00041703399999999999 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.692606+0000", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019470220999999999 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.692707+0000", 2024-09-13T14:44:54.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101011 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.692845+0000", 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013819599999999999 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.700539+0000", 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076937510000000004 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.700699+0000", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015996000000000001 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.700715+0000", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5843999999999999e-05 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.700770+0000", 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5203e-05 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.703210+0000", 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024400530000000002 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119333 8.4 8:261b0eeb:::benchmark_data_smithi017_253885_object27992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:51.904307+0000", 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.0462245600000002, 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025899333, 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.127 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119333 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904307+0000", 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904307+0000", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904308+0000", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.393e-06 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904313+0000", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8509999999999998e-06 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904325+0000", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1803e-05 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.904330+0000", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5509999999999999e-06 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.909167+0000", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048367840000000002 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.909560+0000", 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039291700000000001 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.920303+0000", 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01074286 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.920368+0000", 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5265999999999995e-05 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.920455+0000", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6880999999999997e-05 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.924399+0000", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039444440000000001 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.927302+0000", 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029030739999999998 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.928174+0000", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087194499999999995 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.928297+0000", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012301099999999999 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.930206+0000", 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019085529999999999 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.133 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119396 8.4 8:28e658b5:::benchmark_data_smithi017_253885_object28055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:52.022207+0000", 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.928323813, 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026015184, 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:44:54.134 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119396 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022207+0000", 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022207+0000", 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:44:54.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022209+0000", 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6929999999999999e-06 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022216+0000", 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4489999999999996e-06 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022229+0000", 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3027000000000001e-05 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.022237+0000", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4989999999999994e-06 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.028890+0000", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066532079999999999 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.029571+0000", 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00068116200000000002 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.039348+0000", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097762920000000007 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.039463+0000", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011560799999999999 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.039604+0000", 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141068 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.043589+0000", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039848410000000003 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.044741+0000", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001152124 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.045647+0000", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090575200000000001 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:44:54.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.045711+0000", 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4135999999999997e-05 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:52.048223+0000", 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002511325 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:44:54.140 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:44:54.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:54.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:54.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:54.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:54.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:54.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:54.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:54 smithi017 ceph-mon[103001]: pgmap v1071: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37812/190530 objects degraded (19.846%); 3933/190530 objects misplaced (2.064%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:54.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37812/190530 objects degraded (19.846%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:54.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:54 smithi017 ceph-mon[107556]: pgmap v1071: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37812/190530 objects degraded (19.846%); 3933/190530 objects misplaced (2.064%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:54.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37812/190530 objects degraded (19.846%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:54.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:54.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:54 smithi189 ceph-mon[90992]: pgmap v1071: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37812/190530 objects degraded (19.846%); 3933/190530 objects misplaced (2.064%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37812/190530 objects degraded (19.846%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:54.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:54.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:54.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:55.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:55.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:55.090 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1071,"stamp":"2024-09-13T14:44:53.214938+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4164979382,"num_objects":63510,"num_object_clones":0,"num_object_copies":190530,"num_objects_missing_on_primary":2978,"num_objects_missing":2978,"num_objects_degraded":37812,"num_objects_misplaced":3933,"num_objects_unfound":0,"num_objects_dirty":63510,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":120143,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23289,"num_bytes_recovered":1462274205,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119828,"ondisk_log_size":119828,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16205048,"kb_used_data":11215472,"kb_used_omap":44,"kb_used_meta":4989459,"kb_avail":733756168,"statfs":{"total":767960285184,"available":751366316032,"internally_reserved":0,"allocated":11484643328,"data_stored":11470130973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5109206263},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76021760,"num_objects":-1160,"num_object_clones":0,"num_object_copies":-3480,"num_objects_missing_on_primary":-84,"num_objects_missing":-84,"num_objects_degraded":-226,"num_objects_misplaced":-83,"num_objects_unfound":0,"num_objects_dirty":-1160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1160,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003111"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":581,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.294140+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:44:49.294140+0000","last_peered":"2024-09-13T14:44:49.294140+0000","last_clean":"2024-09-13T14:44:49.294140+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:44:49.294140+0000","last_undegraded":"2024-09-13T14:44:49.294140+0000","last_fullsized":"2024-09-13T14:44:49.294140+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":347,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.293778+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:44:37.293778+0000","last_peered":"2024-09-13T14:44:37.293778+0000","last_clean":"2024-09-13T14:44:37.293778+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:44:37.293778+0000","last_undegraded":"2024-09-13T14:44:37.293778+0000","last_fullsized":"2024-09-13T14:44:37.293778+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.273296+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:44:37.273296+0000","last_peered":"2024-09-13T14:44:37.273296+0000","last_clean":"2024-09-13T14:44:37.273296+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:44:37.273296+0000","last_undegraded":"2024-09-13T14:44:37.273296+0000","last_fullsized":"2024-09-13T14:44:37.273296+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":564,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.293899+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:44:37.293899+0000","last_peered":"2024-09-13T14:44:37.293899+0000","last_clean":"2024-09-13T14:44:37.293899+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:44:37.293899+0000","last_undegraded":"2024-09-13T14:44:37.293899+0000","last_fullsized":"2024-09-13T14:44:37.293899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":565,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294593+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:44:37.294593+0000","last_peered":"2024-09-13T14:44:37.294593+0000","last_clean":"2024-09-13T14:44:37.294593+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:44:37.294593+0000","last_undegraded":"2024-09-13T14:44:37.294593+0000","last_fullsized":"2024-09-13T14:44:37.294593+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":316,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.293767+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:44:37.293767+0000","last_peered":"2024-09-13T14:44:37.293767+0000","last_clean":"2024-09-13T14:44:37.293767+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:44:37.293767+0000","last_undegraded":"2024-09-13T14:44:37.293767+0000","last_fullsized":"2024-09-13T14:44:37.293767+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":581,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293875+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:44:49.293875+0000","last_peered":"2024-09-13T14:44:49.293875+0000","last_clean":"2024-09-13T14:44:49.293875+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:44:49.293875+0000","last_undegraded":"2024-09-13T14:44:49.293875+0000","last_fullsized":"2024-09-13T14:44:49.293875+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":338,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.278881+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:44:37.278881+0000","last_peered":"2024-09-13T14:44:37.278881+0000","last_clean":"2024-09-13T14:44:37.278881+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:44:37.278881+0000","last_undegraded":"2024-09-13T14:44:37.278881+0000","last_fullsized":"2024-09-13T14:44:37.278881+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"349'7409","reported_seq":12803,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:52.036118+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:44:52.036118+0000","last_peered":"2024-09-13T14:44:52.036118+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:44:52.036118+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7409,"log_dups_size":0,"ondisk_log_size":7409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254345216,"num_objects":3881,"num_object_clones":0,"num_object_copies":11643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7409,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1765,"num_bytes_recovered":115671040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2896}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":342,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294169+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:44:37.294169+0000","last_peered":"2024-09-13T14:44:37.294169+0000","last_clean":"2024-09-13T14:44:37.294169+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:44:37.294169+0000","last_undegraded":"2024-09-13T14:44:37.294169+0000","last_fullsized":"2024-09-13T14:44:37.294169+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":261,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272149+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:44:37.272149+0000","last_peered":"2024-09-13T14:44:37.272149+0000","last_clean":"2024-09-13T14:44:37.272149+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:44:37.272149+0000","last_undegraded":"2024-09-13T14:44:37.272149+0000","last_fullsized":"2024-09-13T14:44:37.272149+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"349'7496","reported_seq":17042,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:52.925644+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:44:52.925644+0000","last_peered":"2024-09-13T14:44:52.925644+0000","last_clean":"2024-09-13T14:44:52.925644+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:44:52.925644+0000","last_undegraded":"2024-09-13T14:44:52.925644+0000","last_fullsized":"2024-09-13T14:44:52.925644+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7496,"log_dups_size":0,"ondisk_log_size":7496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7496,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":360,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294268+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:44:37.294268+0000","last_peered":"2024-09-13T14:44:37.294268+0000","last_clean":"2024-09-13T14:44:37.294268+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:44:37.294268+0000","last_undegraded":"2024-09-13T14:44:37.294268+0000","last_fullsized":"2024-09-13T14:44:37.294268+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.280920+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:44:25.280920+0000","last_peered":"2024-09-13T14:44:25.280920+0000","last_clean":"2024-09-13T14:44:25.280920+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:44:25.280920+0000","last_undegraded":"2024-09-13T14:44:25.280920+0000","last_fullsized":"2024-09-13T14:44:25.280920+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"348'7712","reported_seq":7815,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:48.604635+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:44:48.604635+0000","last_peered":"2024-09-13T14:44:48.604635+0000","last_clean":"2024-09-13T14:44:48.604635+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:44:48.604635+0000","last_undegraded":"2024-09-13T14:44:48.604635+0000","last_fullsized":"2024-09-13T14:44:48.604635+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7712,"log_dups_size":0,"ondisk_log_size":7712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267780096,"num_objects":4086,"num_object_clones":0,"num_object_copies":12258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7712,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":314,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294311+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:44:37.294311+0000","last_peered":"2024-09-13T14:44:37.294311+0000","last_clean":"2024-09-13T14:44:37.294311+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:44:37.294311+0000","last_undegraded":"2024-09-13T14:44:37.294311+0000","last_fullsized":"2024-09-13T14:44:37.294311+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":329,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.279124+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:44:37.279124+0000","last_peered":"2024-09-13T14:44:37.279124+0000","last_clean":"2024-09-13T14:44:37.279124+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:44:37.279124+0000","last_undegraded":"2024-09-13T14:44:37.279124+0000","last_fullsized":"2024-09-13T14:44:37.279124+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"349'7532","reported_seq":7637,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.705517+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:44:49.705517+0000","last_peered":"2024-09-13T14:44:49.705517+0000","last_clean":"2024-09-13T14:44:49.705517+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:44:49.705517+0000","last_undegraded":"2024-09-13T14:44:49.705517+0000","last_fullsized":"2024-09-13T14:44:49.705517+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7532,"log_dups_size":0,"ondisk_log_size":7532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263061504,"num_objects":4014,"num_object_clones":0,"num_object_copies":12042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7532,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":277,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272855+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:44:37.272855+0000","last_peered":"2024-09-13T14:44:37.272855+0000","last_clean":"2024-09-13T14:44:37.272855+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:44:37.272855+0000","last_undegraded":"2024-09-13T14:44:37.272855+0000","last_fullsized":"2024-09-13T14:44:37.272855+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":591,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294179+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:44:37.294179+0000","last_peered":"2024-09-13T14:44:37.294179+0000","last_clean":"2024-09-13T14:44:37.294179+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:44:37.294179+0000","last_undegraded":"2024-09-13T14:44:37.294179+0000","last_fullsized":"2024-09-13T14:44:37.294179+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"349'7452","reported_seq":7554,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:50.215875+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:44:50.215875+0000","last_peered":"2024-09-13T14:44:50.215875+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:44:50.215875+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7052,"log_dups_size":0,"ondisk_log_size":7052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262799360,"num_objects":4010,"num_object_clones":0,"num_object_copies":12030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7452,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1623}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":460,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.293650+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:44:37.293650+0000","last_peered":"2024-09-13T14:44:37.293650+0000","last_clean":"2024-09-13T14:44:37.293650+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:44:37.293650+0000","last_undegraded":"2024-09-13T14:44:37.293650+0000","last_fullsized":"2024-09-13T14:44:37.293650+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":591,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293266+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:44:49.293266+0000","last_peered":"2024-09-13T14:44:49.293266+0000","last_clean":"2024-09-13T14:44:49.293266+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:44:49.293266+0000","last_undegraded":"2024-09-13T14:44:49.293266+0000","last_fullsized":"2024-09-13T14:44:49.293266+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281058+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:44:25.281058+0000","last_peered":"2024-09-13T14:44:25.281058+0000","last_clean":"2024-09-13T14:44:25.281058+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:44:25.281058+0000","last_undegraded":"2024-09-13T14:44:25.281058+0000","last_fullsized":"2024-09-13T14:44:25.281058+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"349'7192","reported_seq":7297,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.654574+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:44:49.654574+0000","last_peered":"2024-09-13T14:44:49.654574+0000","last_clean":"2024-09-13T14:44:49.654574+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:44:49.654574+0000","last_undegraded":"2024-09-13T14:44:49.654574+0000","last_fullsized":"2024-09-13T14:44:49.654574+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7192,"log_dups_size":0,"ondisk_log_size":7192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255721472,"num_objects":3902,"num_object_clones":0,"num_object_copies":11706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7192,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":164,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272239+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:44:37.272239+0000","last_peered":"2024-09-13T14:44:37.272239+0000","last_clean":"2024-09-13T14:44:37.272239+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:44:37.272239+0000","last_undegraded":"2024-09-13T14:44:37.272239+0000","last_fullsized":"2024-09-13T14:44:37.272239+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":350,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.278999+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:44:37.278999+0000","last_peered":"2024-09-13T14:44:37.278999+0000","last_clean":"2024-09-13T14:44:37.278999+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:44:37.278999+0000","last_undegraded":"2024-09-13T14:44:37.278999+0000","last_fullsized":"2024-09-13T14:44:37.278999+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":61,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281386+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:44:25.281386+0000","last_peered":"2024-09-13T14:44:25.281386+0000","last_clean":"2024-09-13T14:44:25.281386+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:44:25.281386+0000","last_undegraded":"2024-09-13T14:44:25.281386+0000","last_fullsized":"2024-09-13T14:44:25.281386+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"348'7376","reported_seq":12098,"reported_epoch":348,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:49.025101+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:44:49.025101+0000","last_peered":"2024-09-13T14:44:49.025101+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:44:49.025101+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264503296,"num_objects":4036,"num_object_clones":0,"num_object_copies":12108,"num_objects_missing_on_primary":2978,"num_objects_missing":2978,"num_objects_degraded":3353,"num_objects_misplaced":3933,"num_objects_unfound":0,"num_objects_dirty":4036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7376,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2978},{"shards":"0,3,4","objects":955},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.280688+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:44:25.280688+0000","last_peered":"2024-09-13T14:44:25.280688+0000","last_clean":"2024-09-13T14:44:25.280688+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:44:25.280688+0000","last_undegraded":"2024-09-13T14:44:25.280688+0000","last_fullsized":"2024-09-13T14:44:25.280688+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281149+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:44:25.281149+0000","last_peered":"2024-09-13T14:44:25.281149+0000","last_clean":"2024-09-13T14:44:25.281149+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:44:25.281149+0000","last_undegraded":"2024-09-13T14:44:25.281149+0000","last_fullsized":"2024-09-13T14:44:25.281149+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"349'7286","reported_seq":7391,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:49.881313+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:44:49.881313+0000","last_peered":"2024-09-13T14:44:49.881313+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:44:49.881313+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7286,"log_dups_size":0,"ondisk_log_size":7286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253755392,"num_objects":3872,"num_object_clones":0,"num_object_copies":11616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7286,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1594}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":157,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.277889+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:44:37.277889+0000","last_peered":"2024-09-13T14:44:37.277889+0000","last_clean":"2024-09-13T14:44:37.277889+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:44:37.277889+0000","last_undegraded":"2024-09-13T14:44:37.277889+0000","last_fullsized":"2024-09-13T14:44:37.277889+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":583,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.301317+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:44:49.301317+0000","last_peered":"2024-09-13T14:44:49.301317+0000","last_clean":"2024-09-13T14:44:49.301317+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:44:49.301317+0000","last_undegraded":"2024-09-13T14:44:49.301317+0000","last_fullsized":"2024-09-13T14:44:49.301317+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":545,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303683+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:44:49.303683+0000","last_peered":"2024-09-13T14:44:49.303683+0000","last_clean":"2024-09-13T14:44:49.303683+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:44:49.303683+0000","last_undegraded":"2024-09-13T14:44:49.303683+0000","last_fullsized":"2024-09-13T14:44:49.303683+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"349'7360","reported_seq":7421,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:52.949810+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:44:52.949810+0000","last_peered":"2024-09-13T14:44:52.949810+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:44:52.949810+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252968960,"num_objects":3860,"num_object_clones":0,"num_object_copies":11580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7360,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1636}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":451,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.296118+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:44:37.296118+0000","last_peered":"2024-09-13T14:44:37.296118+0000","last_clean":"2024-09-13T14:44:37.296118+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:44:37.296118+0000","last_undegraded":"2024-09-13T14:44:37.296118+0000","last_fullsized":"2024-09-13T14:44:37.296118+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":873,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272812+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:44:37.272812+0000","last_peered":"2024-09-13T14:44:37.272812+0000","last_clean":"2024-09-13T14:44:37.272812+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:44:37.272812+0000","last_undegraded":"2024-09-13T14:44:37.272812+0000","last_fullsized":"2024-09-13T14:44:37.272812+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":537,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293464+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:44:49.293464+0000","last_peered":"2024-09-13T14:44:49.293464+0000","last_clean":"2024-09-13T14:44:49.293464+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:44:49.293464+0000","last_undegraded":"2024-09-13T14:44:49.293464+0000","last_fullsized":"2024-09-13T14:44:49.293464+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":315,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.279701+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:44:37.279701+0000","last_peered":"2024-09-13T14:44:37.279701+0000","last_clean":"2024-09-13T14:44:37.279701+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:44:37.279701+0000","last_undegraded":"2024-09-13T14:44:37.279701+0000","last_fullsized":"2024-09-13T14:44:37.279701+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"349'7535","reported_seq":7638,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:49.701464+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:44:49.701464+0000","last_peered":"2024-09-13T14:44:49.701464+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:44:49.701464+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262995968,"num_objects":4013,"num_object_clones":0,"num_object_copies":12039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7535,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1667}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":390,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303072+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:44:49.303072+0000","last_peered":"2024-09-13T14:44:49.303072+0000","last_clean":"2024-09-13T14:44:49.303072+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:44:49.303072+0000","last_undegraded":"2024-09-13T14:44:49.303072+0000","last_fullsized":"2024-09-13T14:44:49.303072+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":739,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.296070+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:44:37.296070+0000","last_peered":"2024-09-13T14:44:37.296070+0000","last_clean":"2024-09-13T14:44:37.296070+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:44:37.296070+0000","last_undegraded":"2024-09-13T14:44:37.296070+0000","last_fullsized":"2024-09-13T14:44:37.296070+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":584,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294203+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:44:37.294203+0000","last_peered":"2024-09-13T14:44:37.294203+0000","last_clean":"2024-09-13T14:44:37.294203+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:44:37.294203+0000","last_undegraded":"2024-09-13T14:44:37.294203+0000","last_fullsized":"2024-09-13T14:44:37.294203+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":588,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294713+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:44:37.294713+0000","last_peered":"2024-09-13T14:44:37.294713+0000","last_clean":"2024-09-13T14:44:37.294713+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:44:37.294713+0000","last_undegraded":"2024-09-13T14:44:37.294713+0000","last_fullsized":"2024-09-13T14:44:37.294713+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"349'7548","reported_seq":17194,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:50.105462+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:44:50.105462+0000","last_peered":"2024-09-13T14:44:50.105462+0000","last_clean":"2024-09-13T14:44:50.105462+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:44:50.105462+0000","last_undegraded":"2024-09-13T14:44:50.105462+0000","last_fullsized":"2024-09-13T14:44:50.105462+0000","mapping_epoch":330,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269746176,"num_objects":4116,"num_object_clones":0,"num_object_copies":12348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7548,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":480,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.293681+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:44:37.293681+0000","last_peered":"2024-09-13T14:44:37.293681+0000","last_clean":"2024-09-13T14:44:37.293681+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:44:37.293681+0000","last_undegraded":"2024-09-13T14:44:37.293681+0000","last_fullsized":"2024-09-13T14:44:37.293681+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":554,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293605+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:44:49.293605+0000","last_peered":"2024-09-13T14:44:49.293605+0000","last_clean":"2024-09-13T14:44:49.293605+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:44:49.293605+0000","last_undegraded":"2024-09-13T14:44:49.293605+0000","last_fullsized":"2024-09-13T14:44:49.293605+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":523,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.302661+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:44:49.302661+0000","last_peered":"2024-09-13T14:44:49.302661+0000","last_clean":"2024-09-13T14:44:49.302661+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:44:49.302661+0000","last_undegraded":"2024-09-13T14:44:49.302661+0000","last_fullsized":"2024-09-13T14:44:49.302661+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":98,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272997+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:44:37.272997+0000","last_peered":"2024-09-13T14:44:37.272997+0000","last_clean":"2024-09-13T14:44:37.272997+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:44:37.272997+0000","last_undegraded":"2024-09-13T14:44:37.272997+0000","last_fullsized":"2024-09-13T14:44:37.272997+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"349'7281","reported_seq":7385,"reported_epoch":349,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:44:50.275839+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:44:50.275839+0000","last_peered":"2024-09-13T14:44:50.275839+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:44:50.275839+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801664,"num_objects":3949,"num_object_clones":0,"num_object_copies":11847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7281,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1560}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":157,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272586+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:44:37.272586+0000","last_peered":"2024-09-13T14:44:37.272586+0000","last_clean":"2024-09-13T14:44:37.272586+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:44:37.272586+0000","last_undegraded":"2024-09-13T14:44:37.272586+0000","last_fullsized":"2024-09-13T14:44:37.272586+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":964,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.277961+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:44:37.277961+0000","last_peered":"2024-09-13T14:44:37.277961+0000","last_clean":"2024-09-13T14:44:37.277961+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:44:37.277961+0000","last_undegraded":"2024-09-13T14:44:37.277961+0000","last_fullsized":"2024-09-13T14:44:37.277961+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281179+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:44:25.281179+0000","last_peered":"2024-09-13T14:44:25.281179+0000","last_clean":"2024-09-13T14:44:25.281179+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:44:25.281179+0000","last_undegraded":"2024-09-13T14:44:25.281179+0000","last_fullsized":"2024-09-13T14:44:25.281179+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":543,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.302965+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:44:49.302965+0000","last_peered":"2024-09-13T14:44:49.302965+0000","last_clean":"2024-09-13T14:44:49.302965+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:44:49.302965+0000","last_undegraded":"2024-09-13T14:44:49.302965+0000","last_fullsized":"2024-09-13T14:44:49.302965+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"348'7442","reported_seq":16825,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:48.785426+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:44:48.785426+0000","last_peered":"2024-09-13T14:44:48.785426+0000","last_clean":"2024-09-13T14:44:48.785426+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:44:48.785426+0000","last_undegraded":"2024-09-13T14:44:48.785426+0000","last_fullsized":"2024-09-13T14:44:48.785426+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7442,"log_dups_size":0,"ondisk_log_size":7442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256770048,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7442,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":348,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.281611+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:44:37.281611+0000","last_peered":"2024-09-13T14:44:37.281611+0000","last_clean":"2024-09-13T14:44:37.281611+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:44:37.281611+0000","last_undegraded":"2024-09-13T14:44:37.281611+0000","last_fullsized":"2024-09-13T14:44:37.281611+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":294,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.273184+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:44:37.273184+0000","last_peered":"2024-09-13T14:44:37.273184+0000","last_clean":"2024-09-13T14:44:37.273184+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:44:37.273184+0000","last_undegraded":"2024-09-13T14:44:37.273184+0000","last_fullsized":"2024-09-13T14:44:37.273184+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1122,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303094+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:44:49.303094+0000","last_peered":"2024-09-13T14:44:49.303094+0000","last_clean":"2024-09-13T14:44:49.303094+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:44:49.303094+0000","last_undegraded":"2024-09-13T14:44:49.303094+0000","last_fullsized":"2024-09-13T14:44:49.303094+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":524,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.306342+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:44:49.306342+0000","last_peered":"2024-09-13T14:44:49.306342+0000","last_clean":"2024-09-13T14:44:49.306342+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:44:49.306342+0000","last_undegraded":"2024-09-13T14:44:49.306342+0000","last_fullsized":"2024-09-13T14:44:49.306342+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"349'7307","reported_seq":7412,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:50.418833+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:44:50.418833+0000","last_peered":"2024-09-13T14:44:50.418833+0000","last_clean":"2024-09-13T14:44:50.418833+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:44:50.418833+0000","last_undegraded":"2024-09-13T14:44:50.418833+0000","last_fullsized":"2024-09-13T14:44:50.418833+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255393792,"num_objects":3897,"num_object_clones":0,"num_object_copies":11691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7307,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":348,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.279492+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:44:37.279492+0000","last_peered":"2024-09-13T14:44:37.279492+0000","last_clean":"2024-09-13T14:44:37.279492+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:44:37.279492+0000","last_undegraded":"2024-09-13T14:44:37.279492+0000","last_fullsized":"2024-09-13T14:44:37.279492+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":533,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303040+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:44:49.303040+0000","last_peered":"2024-09-13T14:44:49.303040+0000","last_clean":"2024-09-13T14:44:49.303040+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:44:49.303040+0000","last_undegraded":"2024-09-13T14:44:49.303040+0000","last_fullsized":"2024-09-13T14:44:49.303040+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":743,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294381+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:44:37.294381+0000","last_peered":"2024-09-13T14:44:37.294381+0000","last_clean":"2024-09-13T14:44:37.294381+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:44:37.294381+0000","last_undegraded":"2024-09-13T14:44:37.294381+0000","last_fullsized":"2024-09-13T14:44:37.294381+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":567,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294110+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:44:37.294110+0000","last_peered":"2024-09-13T14:44:37.294110+0000","last_clean":"2024-09-13T14:44:37.294110+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:44:37.294110+0000","last_undegraded":"2024-09-13T14:44:37.294110+0000","last_fullsized":"2024-09-13T14:44:37.294110+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"348'7675","reported_seq":7778,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:48.596257+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:44:48.596257+0000","last_peered":"2024-09-13T14:44:48.596257+0000","last_clean":"2024-09-13T14:44:48.596257+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:44:48.596257+0000","last_undegraded":"2024-09-13T14:44:48.596257+0000","last_fullsized":"2024-09-13T14:44:48.596257+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7675,"log_dups_size":0,"ondisk_log_size":7675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263127040,"num_objects":4015,"num_object_clones":0,"num_object_copies":12045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7675,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281225+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:44:25.281225+0000","last_peered":"2024-09-13T14:44:25.281225+0000","last_clean":"2024-09-13T14:44:25.281225+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:44:25.281225+0000","last_undegraded":"2024-09-13T14:44:25.281225+0000","last_fullsized":"2024-09-13T14:44:25.281225+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":569,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.294128+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:44:49.294128+0000","last_peered":"2024-09-13T14:44:49.294128+0000","last_clean":"2024-09-13T14:44:49.294128+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:44:49.294128+0000","last_undegraded":"2024-09-13T14:44:49.294128+0000","last_fullsized":"2024-09-13T14:44:49.294128+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":909,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303388+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:44:49.303388+0000","last_peered":"2024-09-13T14:44:49.303388+0000","last_clean":"2024-09-13T14:44:49.303388+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:44:49.303388+0000","last_undegraded":"2024-09-13T14:44:49.303388+0000","last_fullsized":"2024-09-13T14:44:49.303388+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":567,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.301210+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:44:49.301210+0000","last_peered":"2024-09-13T14:44:49.301210+0000","last_clean":"2024-09-13T14:44:49.301210+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:44:49.301210+0000","last_undegraded":"2024-09-13T14:44:49.301210+0000","last_fullsized":"2024-09-13T14:44:49.301210+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"349'7593","reported_seq":18953,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:52.046041+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:44:52.046041+0000","last_peered":"2024-09-13T14:44:52.046041+0000","last_clean":"2024-09-13T14:44:52.046041+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:44:52.046041+0000","last_undegraded":"2024-09-13T14:44:52.046041+0000","last_fullsized":"2024-09-13T14:44:52.046041+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7593,"log_dups_size":0,"ondisk_log_size":7593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901144,"num_objects":3921,"num_object_clones":0,"num_object_copies":11763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3921,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7593,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":421,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.294099+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:44:49.294099+0000","last_peered":"2024-09-13T14:44:49.294099+0000","last_clean":"2024-09-13T14:44:49.294099+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:44:49.294099+0000","last_undegraded":"2024-09-13T14:44:49.294099+0000","last_fullsized":"2024-09-13T14:44:49.294099+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":602,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294539+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:44:37.294539+0000","last_peered":"2024-09-13T14:44:37.294539+0000","last_clean":"2024-09-13T14:44:37.294539+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:44:37.294539+0000","last_undegraded":"2024-09-13T14:44:37.294539+0000","last_fullsized":"2024-09-13T14:44:37.294539+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":767,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.281702+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:44:37.281702+0000","last_peered":"2024-09-13T14:44:37.281702+0000","last_clean":"2024-09-13T14:44:37.281702+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:44:37.281702+0000","last_undegraded":"2024-09-13T14:44:37.281702+0000","last_fullsized":"2024-09-13T14:44:37.281702+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.280773+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:44:25.280773+0000","last_peered":"2024-09-13T14:44:25.280773+0000","last_clean":"2024-09-13T14:44:25.280773+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:44:25.280773+0000","last_undegraded":"2024-09-13T14:44:25.280773+0000","last_fullsized":"2024-09-13T14:44:25.280773+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281246+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:44:25.281246+0000","last_peered":"2024-09-13T14:44:25.281246+0000","last_clean":"2024-09-13T14:44:25.281246+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:44:25.281246+0000","last_undegraded":"2024-09-13T14:44:25.281246+0000","last_fullsized":"2024-09-13T14:44:25.281246+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":57,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281108+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:44:25.281108+0000","last_peered":"2024-09-13T14:44:25.281108+0000","last_clean":"2024-09-13T14:44:25.281108+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:44:25.281108+0000","last_undegraded":"2024-09-13T14:44:25.281108+0000","last_fullsized":"2024-09-13T14:44:25.281108+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":521,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.306181+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:44:49.306181+0000","last_peered":"2024-09-13T14:44:49.306181+0000","last_clean":"2024-09-13T14:44:49.306181+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:44:49.306181+0000","last_undegraded":"2024-09-13T14:44:49.306181+0000","last_fullsized":"2024-09-13T14:44:49.306181+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":348,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294199+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:44:37.294199+0000","last_peered":"2024-09-13T14:44:37.294199+0000","last_clean":"2024-09-13T14:44:37.294199+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:44:37.294199+0000","last_undegraded":"2024-09-13T14:44:37.294199+0000","last_fullsized":"2024-09-13T14:44:37.294199+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":314,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272736+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:44:37.272736+0000","last_peered":"2024-09-13T14:44:37.272736+0000","last_clean":"2024-09-13T14:44:37.272736+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:44:37.272736+0000","last_undegraded":"2024-09-13T14:44:37.272736+0000","last_fullsized":"2024-09-13T14:44:37.272736+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":525,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293022+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:44:49.293022+0000","last_peered":"2024-09-13T14:44:49.293022+0000","last_clean":"2024-09-13T14:44:49.293022+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:44:49.293022+0000","last_undegraded":"2024-09-13T14:44:49.293022+0000","last_fullsized":"2024-09-13T14:44:49.293022+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":575,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294297+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:44:37.294297+0000","last_peered":"2024-09-13T14:44:37.294297+0000","last_clean":"2024-09-13T14:44:37.294297+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:44:37.294297+0000","last_undegraded":"2024-09-13T14:44:37.294297+0000","last_fullsized":"2024-09-13T14:44:37.294297+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":581,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.301219+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:44:49.301219+0000","last_peered":"2024-09-13T14:44:49.301219+0000","last_clean":"2024-09-13T14:44:49.301219+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:44:49.301219+0000","last_undegraded":"2024-09-13T14:44:49.301219+0000","last_fullsized":"2024-09-13T14:44:49.301219+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":346,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.281661+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:44:37.281661+0000","last_peered":"2024-09-13T14:44:37.281661+0000","last_clean":"2024-09-13T14:44:37.281661+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:44:37.281661+0000","last_undegraded":"2024-09-13T14:44:37.281661+0000","last_fullsized":"2024-09-13T14:44:37.281661+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":599,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.301269+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:44:49.301269+0000","last_peered":"2024-09-13T14:44:49.301269+0000","last_clean":"2024-09-13T14:44:49.301269+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:44:49.301269+0000","last_undegraded":"2024-09-13T14:44:49.301269+0000","last_fullsized":"2024-09-13T14:44:49.301269+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":567,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294639+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:44:37.294639+0000","last_peered":"2024-09-13T14:44:37.294639+0000","last_clean":"2024-09-13T14:44:37.294639+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:44:37.294639+0000","last_undegraded":"2024-09-13T14:44:37.294639+0000","last_fullsized":"2024-09-13T14:44:37.294639+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":546,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.302731+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:44:49.302731+0000","last_peered":"2024-09-13T14:44:49.302731+0000","last_clean":"2024-09-13T14:44:49.302731+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:44:49.302731+0000","last_undegraded":"2024-09-13T14:44:49.302731+0000","last_fullsized":"2024-09-13T14:44:49.302731+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":595,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293521+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:44:49.293521+0000","last_peered":"2024-09-13T14:44:49.293521+0000","last_clean":"2024-09-13T14:44:49.293521+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:44:49.293521+0000","last_undegraded":"2024-09-13T14:44:49.293521+0000","last_fullsized":"2024-09-13T14:44:49.293521+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":46,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.287003+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:44:25.287003+0000","last_peered":"2024-09-13T14:44:25.287003+0000","last_clean":"2024-09-13T14:44:25.287003+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:44:25.287003+0000","last_undegraded":"2024-09-13T14:44:25.287003+0000","last_fullsized":"2024-09-13T14:44:25.287003+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":492,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.303174+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:44:49.303174+0000","last_peered":"2024-09-13T14:44:49.303174+0000","last_clean":"2024-09-13T14:44:49.303174+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:44:49.303174+0000","last_undegraded":"2024-09-13T14:44:49.303174+0000","last_fullsized":"2024-09-13T14:44:49.303174+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":571,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294065+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:44:37.294065+0000","last_peered":"2024-09-13T14:44:37.294065+0000","last_clean":"2024-09-13T14:44:37.294065+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:44:37.294065+0000","last_undegraded":"2024-09-13T14:44:37.294065+0000","last_fullsized":"2024-09-13T14:44:37.294065+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.281099+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:44:25.281099+0000","last_peered":"2024-09-13T14:44:25.281099+0000","last_clean":"2024-09-13T14:44:25.281099+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:44:25.281099+0000","last_undegraded":"2024-09-13T14:44:25.281099+0000","last_fullsized":"2024-09-13T14:44:25.281099+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":314,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.272908+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:44:37.272908+0000","last_peered":"2024-09-13T14:44:37.272908+0000","last_clean":"2024-09-13T14:44:37.272908+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:44:37.272908+0000","last_undegraded":"2024-09-13T14:44:37.272908+0000","last_fullsized":"2024-09-13T14:44:37.272908+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":46,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.286912+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:44:25.286912+0000","last_peered":"2024-09-13T14:44:25.286912+0000","last_clean":"2024-09-13T14:44:25.286912+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:44:25.286912+0000","last_undegraded":"2024-09-13T14:44:25.286912+0000","last_fullsized":"2024-09-13T14:44:25.286912+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":280,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.273373+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:44:37.273373+0000","last_peered":"2024-09-13T14:44:37.273373+0000","last_clean":"2024-09-13T14:44:37.273373+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:44:37.273373+0000","last_undegraded":"2024-09-13T14:44:37.273373+0000","last_fullsized":"2024-09-13T14:44:37.273373+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":591,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293689+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:44:49.293689+0000","last_peered":"2024-09-13T14:44:49.293689+0000","last_clean":"2024-09-13T14:44:49.293689+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:44:49.293689+0000","last_undegraded":"2024-09-13T14:44:49.293689+0000","last_fullsized":"2024-09-13T14:44:49.293689+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":578,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293964+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:44:49.293964+0000","last_peered":"2024-09-13T14:44:49.293964+0000","last_clean":"2024-09-13T14:44:49.293964+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:44:49.293964+0000","last_undegraded":"2024-09-13T14:44:49.293964+0000","last_fullsized":"2024-09-13T14:44:49.293964+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":347,"state":"active+clean","last_fresh":"2024-09-13T14:44:25.280650+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:44:25.280650+0000","last_peered":"2024-09-13T14:44:25.280650+0000","last_clean":"2024-09-13T14:44:25.280650+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:44:25.280650+0000","last_undegraded":"2024-09-13T14:44:25.280650+0000","last_fullsized":"2024-09-13T14:44:25.280650+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":581,"reported_epoch":348,"state":"active+clean","last_fresh":"2024-09-13T14:44:37.294289+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:44:37.294289+0000","last_peered":"2024-09-13T14:44:37.294289+0000","last_clean":"2024-09-13T14:44:37.294289+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:44:37.294289+0000","last_undegraded":"2024-09-13T14:44:37.294289+0000","last_fullsized":"2024-09-13T14:44:37.294289+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4160290840,"num_objects":63482,"num_object_clones":0,"num_object_copies":190446,"num_objects_missing_on_primary":2978,"num_objects_missing":2978,"num_objects_degraded":37812,"num_objects_misplaced":3933,"num_objects_unfound":0,"num_objects_dirty":63482,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119196,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23232,"num_bytes_recovered":1462108160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11425558528,"data_stored":11425546312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117696,"ondisk_log_size":117696,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397905,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2232012,"kb_used_data":1340164,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91513140,"statfs":{"total":95995035648,"available":93709455360,"internally_reserved":0,"allocated":1372327936,"data_stored":1370496374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594133,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208568,"kb_used_data":1622832,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91536584,"statfs":{"total":95995035648,"available":93733462016,"internally_reserved":0,"allocated":1661779968,"data_stored":1659968160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665587,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639224,"kb_used_data":302512,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93105928,"statfs":{"total":95995035648,"available":95340470272,"internally_reserved":0,"allocated":309772288,"data_stored":307956811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986588,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1811124,"kb_used_data":907116,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":91934028,"statfs":{"total":95995035648,"available":94140444672,"internally_reserved":0,"allocated":928886784,"data_stored":927071167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697148,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2640632,"kb_used_data":1900524,"kb_used_omap":4,"kb_used_meta":740091,"kb_avail":91104520,"statfs":{"total":95995035648,"available":93291028480,"internally_reserved":0,"allocated":1946136576,"data_stored":1944325252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":757853259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860675,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2440956,"kb_used_data":1815540,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91304196,"statfs":{"total":95995035648,"available":93495496704,"internally_reserved":0,"allocated":1859112960,"data_stored":1857293982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020346,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2085748,"kb_used_data":1790504,"kb_used_omap":9,"kb_used_meta":295222,"kb_avail":91659404,"statfs":{"total":95995035648,"available":93859229696,"internally_reserved":0,"allocated":1833476096,"data_stored":1831668245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":302308165},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187736,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2146784,"kb_used_data":1536280,"kb_used_omap":7,"kb_used_meta":610488,"kb_avail":91598368,"statfs":{"total":95995035648,"available":93796728832,"internally_reserved":0,"allocated":1573150720,"data_stored":1571350982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":625140043},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1562771456,"data_stored":1562771456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1824129024,"data_stored":1824129024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1850478592,"data_stored":1850474520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1939144704,"data_stored":1939144704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":924254208,"data_stored":924254208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":304746496,"data_stored":304742424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1656750080,"data_stored":1656750080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1363283968,"data_stored":1363279896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:44:55.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:55.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:55.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:55.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:55 smithi017 ceph-mon[107556]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:55.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:55 smithi017 ceph-mon[103001]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:55.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:55.493 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:55.493 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:55.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:55.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:55.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:55 smithi189 ceph-mon[90992]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:44:55.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:55.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:55.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:55.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:56.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:56.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:56.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:56.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:44:56.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:56.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:56.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:56.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:56.491 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:56 smithi017 ceph-mon[103001]: pgmap v1072: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37771/189651 objects degraded (19.916%); 3890/189651 objects misplaced (2.051%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:56.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:56 smithi017 ceph-mon[107556]: pgmap v1072: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37771/189651 objects degraded (19.916%); 3890/189651 objects misplaced (2.051%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:56.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:56.562 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:56.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:56 smithi189 ceph-mon[90992]: pgmap v1072: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37771/189651 objects degraded (19.916%); 3890/189651 objects misplaced (2.051%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:44:56.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.813 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:56.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:57.037 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:57.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:57.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:57.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:57.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:57.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:57.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:57.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:57.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:44:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:44:57] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-13T14:44:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:57.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:57.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:57.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:57.684 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:57.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:57.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:57.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:57.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:57.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:58.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:58.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:58.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:58.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:44:58.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:58.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:58.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:58.306 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:58 smithi017 ceph-mon[103001]: pgmap v1073: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 37703/189123 objects degraded (19.936%); 3890/189123 objects misplaced (2.057%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:44:58.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:58.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:58.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:58.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:58.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:58.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:58.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:58.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:58 smithi017 ceph-mon[107556]: pgmap v1073: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 37703/189123 objects degraded (19.936%); 3890/189123 objects misplaced (2.057%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:44:58.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:58 smithi189 ceph-mon[90992]: pgmap v1073: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 37703/189123 objects degraded (19.936%); 3890/189123 objects misplaced (2.057%); 992 KiB/s, 15 objects/s recovering 2024-09-13T14:44:58.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:58.807 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:58.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:58.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:59.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:59.057 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:59.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:59.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:59.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:59.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:59.373 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:44:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37703/189123 objects degraded (19.936%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:59.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:44:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37703/189123 objects degraded (19.936%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:59.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:59.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:59.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:59.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:59.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:59.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:59.699 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:44:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37703/189123 objects degraded (19.936%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:44:59.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:59.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:59.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:59.811 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:44:59.811 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:59.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:44:59.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:59.833 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:44:59.834 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:59.915 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:59.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:44:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:00.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:00.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:00.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:00.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:00.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:00.320 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:45:00.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:00.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:00.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:00 smithi189 ceph-mon[90992]: pgmap v1074: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37660/188331 objects degraded (19.997%); 3847/188331 objects misplaced (2.043%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3886854426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1059547660' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:00 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:00.562 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:00.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:00.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:00.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[103001]: pgmap v1074: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37660/188331 objects degraded (19.997%); 3847/188331 objects misplaced (2.043%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3886854426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1059547660' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[107556]: pgmap v1074: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 37660/188331 objects degraded (19.997%); 3847/188331 objects misplaced (2.043%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3886854426' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1059547660' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:00 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:00.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:00.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:01.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:01.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:01.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:01.182 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:01.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:01.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:01.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:01.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:01.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:01.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:01.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:01.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[103001]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:45:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:01 smithi017 ceph-mon[107556]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:45:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:01 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:01.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:01 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:01 smithi189 ceph-mon[90992]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:45:01.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:01.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:01.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:01.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:02.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:02.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:02.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:02.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:02.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:02.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:02.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:02.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:02.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:02.526 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:02.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:02.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:02.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:02 smithi017 ceph-mon[103001]: pgmap v1076: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37660/187281 objects degraded (20.109%); 3847/187281 objects misplaced (2.054%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:02.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:02 smithi017 ceph-mon[107556]: pgmap v1076: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37660/187281 objects degraded (20.109%); 3847/187281 objects misplaced (2.054%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:02.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:02 smithi189 ceph-mon[90992]: pgmap v1076: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 37660/187281 objects degraded (20.109%); 3847/187281 objects misplaced (2.054%); 1011 KiB/s, 15 objects/s recovering 2024-09-13T14:45:02.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:02.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:02.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:02.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:02.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:03.032 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:03.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:03.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:03.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:03.197 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:03.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:03.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:03.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:03.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:03.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:03.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:03.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:03.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:03.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:03.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:03.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:03.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:04.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:04.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:04.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:04.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:04.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:04.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:04.400 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:04.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:04.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.674 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:04.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:04.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:45:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:04 smithi017 ceph-mon[103001]: pgmap v1077: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 37566/186798 objects degraded (20.110%); 3847/186798 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37566/186798 objects degraded (20.110%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:04 smithi017 ceph-mon[107556]: pgmap v1077: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 37566/186798 objects degraded (20.110%); 3847/186798 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37566/186798 objects degraded (20.110%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:04 smithi189 ceph-mon[90992]: pgmap v1077: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 37566/186798 objects degraded (20.110%); 3847/186798 objects misplaced (2.059%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37566/186798 objects degraded (20.110%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:04.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:04.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:04.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:04.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:05.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:05.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:05.055 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:05.055 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:05.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:05.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:05.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:05.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:05.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:05.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:05.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:05.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:05.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:05.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:05.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:05.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:45:05.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:05.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:05.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:05.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:05.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:06.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:06.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:06.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:06.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:06.272 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:06.275 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:06.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:06.300 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:06.333 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:06.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:06.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:06.493 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:06.493 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:45:06.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:06.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:06.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:06 smithi017 ceph-mon[103001]: pgmap v1078: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37523/185802 objects degraded (20.195%); 3805/185802 objects misplaced (2.048%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:06.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:06 smithi017 ceph-mon[107556]: pgmap v1078: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37523/185802 objects degraded (20.195%); 3805/185802 objects misplaced (2.048%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:06.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:06.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:06 smithi189 ceph-mon[90992]: pgmap v1078: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37523/185802 objects degraded (20.195%); 3805/185802 objects misplaced (2.048%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:06.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:06.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:06.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:06.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:07.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:07.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:07.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:07.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:07.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:07.307 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:07.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:07.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:07.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:07.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:07.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:07 smithi017 ceph-mon[103001]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:07 smithi017 ceph-mon[107556]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:07] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-13T14:45:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:07 smithi189 ceph-mon[90992]: from='client.36369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:07.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:07.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:07.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:07.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:08.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:08.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:08.156 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:08.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:08.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:08.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:08.342 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:08 smithi017 ceph-mon[103001]: pgmap v1079: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37453/185259 objects degraded (20.217%); 3805/185259 objects misplaced (2.054%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:08.342 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:08.342 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:08 smithi017 ceph-mon[107556]: pgmap v1079: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37453/185259 objects degraded (20.217%); 3805/185259 objects misplaced (2.054%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:08.342 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:08.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:08.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:08.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:08.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:08.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:08.625 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:08 smithi189 ceph-mon[90992]: pgmap v1079: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 37453/185259 objects degraded (20.217%); 3805/185259 objects misplaced (2.054%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:08.625 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:08.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:08.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:08.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:08.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:08.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:08.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:09.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:09.045 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:09.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:09.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:09.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:09.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:09.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:09.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:09.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37453/185259 objects degraded (20.217%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37453/185259 objects degraded (20.217%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:09.480 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37453/185259 objects degraded (20.217%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:09.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:09.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:09.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:09.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:09.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:09.701 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1078,"stamp":"2024-09-13T14:45:05.218220+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4061694646,"num_objects":61934,"num_object_clones":0,"num_object_copies":185802,"num_objects_missing_on_primary":2851,"num_objects_missing":2851,"num_objects_degraded":37523,"num_objects_misplaced":3805,"num_objects_unfound":0,"num_objects_dirty":61934,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":121719,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23498,"num_bytes_recovered":1475971229,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120804,"ondisk_log_size":120804,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15927116,"kb_used_data":10962496,"kb_used_omap":44,"kb_used_meta":4964499,"kb_avail":734034100,"statfs":{"total":767960285184,"available":751650918400,"internally_reserved":0,"allocated":11225595904,"data_stored":11211012926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5083647223},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-84082688,"num_objects":-1283,"num_object_clones":0,"num_object_copies":-3849,"num_objects_missing_on_primary":-86,"num_objects_missing":-86,"num_objects_degraded":-248,"num_objects_misplaced":-85,"num_objects_unfound":0,"num_objects_dirty":-1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1283,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002662"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":584,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317086+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:45:00.317086+0000","last_peered":"2024-09-13T14:45:00.317086+0000","last_clean":"2024-09-13T14:45:00.317086+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:45:00.317086+0000","last_undegraded":"2024-09-13T14:45:00.317086+0000","last_fullsized":"2024-09-13T14:45:00.317086+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":348,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324757+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:44:49.324757+0000","last_peered":"2024-09-13T14:44:49.324757+0000","last_clean":"2024-09-13T14:44:49.324757+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:44:49.324757+0000","last_undegraded":"2024-09-13T14:44:49.324757+0000","last_fullsized":"2024-09-13T14:44:49.324757+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290491+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:44:49.290491+0000","last_peered":"2024-09-13T14:44:49.290491+0000","last_clean":"2024-09-13T14:44:49.290491+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:44:49.290491+0000","last_undegraded":"2024-09-13T14:44:49.290491+0000","last_fullsized":"2024-09-13T14:44:49.290491+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":569,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.324387+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:45:00.324387+0000","last_peered":"2024-09-13T14:45:00.324387+0000","last_clean":"2024-09-13T14:45:00.324387+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:45:00.324387+0000","last_undegraded":"2024-09-13T14:45:00.324387+0000","last_fullsized":"2024-09-13T14:45:00.324387+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":569,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328854+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:45:00.328854+0000","last_peered":"2024-09-13T14:45:00.328854+0000","last_clean":"2024-09-13T14:45:00.328854+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:45:00.328854+0000","last_undegraded":"2024-09-13T14:45:00.328854+0000","last_fullsized":"2024-09-13T14:45:00.328854+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":317,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324616+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:44:49.324616+0000","last_peered":"2024-09-13T14:44:49.324616+0000","last_clean":"2024-09-13T14:44:49.324616+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:44:49.324616+0000","last_undegraded":"2024-09-13T14:44:49.324616+0000","last_fullsized":"2024-09-13T14:44:49.324616+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":583,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317036+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:45:00.317036+0000","last_peered":"2024-09-13T14:45:00.317036+0000","last_clean":"2024-09-13T14:45:00.317036+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:45:00.317036+0000","last_undegraded":"2024-09-13T14:45:00.317036+0000","last_fullsized":"2024-09-13T14:45:00.317036+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":339,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291199+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:44:49.291199+0000","last_peered":"2024-09-13T14:44:49.291199+0000","last_clean":"2024-09-13T14:44:49.291199+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:44:49.291199+0000","last_undegraded":"2024-09-13T14:44:49.291199+0000","last_fullsized":"2024-09-13T14:44:49.291199+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"350'7491","reported_seq":13130,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:02.005103+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:45:02.005103+0000","last_peered":"2024-09-13T14:45:02.005103+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:45:02.005103+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7491,"log_dups_size":0,"ondisk_log_size":7491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248971264,"num_objects":3799,"num_object_clones":0,"num_object_copies":11397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6513,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1846,"num_bytes_recovered":120979456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2815}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":343,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324979+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:44:49.324979+0000","last_peered":"2024-09-13T14:44:49.324979+0000","last_clean":"2024-09-13T14:44:49.324979+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:44:49.324979+0000","last_undegraded":"2024-09-13T14:44:49.324979+0000","last_fullsized":"2024-09-13T14:44:49.324979+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290478+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:44:49.290478+0000","last_peered":"2024-09-13T14:44:49.290478+0000","last_clean":"2024-09-13T14:44:49.290478+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:44:49.290478+0000","last_undegraded":"2024-09-13T14:44:49.290478+0000","last_fullsized":"2024-09-13T14:44:49.290478+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"350'7574","reported_seq":17121,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:02.778876+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:45:02.778876+0000","last_peered":"2024-09-13T14:45:02.778876+0000","last_clean":"2024-09-13T14:45:02.778876+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:45:02.778876+0000","last_undegraded":"2024-09-13T14:45:02.778876+0000","last_fullsized":"2024-09-13T14:45:02.778876+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256507904,"num_objects":3914,"num_object_clones":0,"num_object_copies":11742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7574,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":361,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.325026+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:44:49.325026+0000","last_peered":"2024-09-13T14:44:49.325026+0000","last_clean":"2024-09-13T14:44:49.325026+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:44:49.325026+0000","last_undegraded":"2024-09-13T14:44:49.325026+0000","last_fullsized":"2024-09-13T14:44:49.325026+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.292886+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:44:49.292886+0000","last_peered":"2024-09-13T14:44:49.292886+0000","last_clean":"2024-09-13T14:44:49.292886+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:44:49.292886+0000","last_undegraded":"2024-09-13T14:44:49.292886+0000","last_fullsized":"2024-09-13T14:44:49.292886+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"350'7828","reported_seq":7935,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:03.142277+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:45:03.142277+0000","last_peered":"2024-09-13T14:45:03.142277+0000","last_clean":"2024-09-13T14:45:03.142277+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:45:03.142277+0000","last_undegraded":"2024-09-13T14:45:03.142277+0000","last_fullsized":"2024-09-13T14:45:03.142277+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7828,"log_dups_size":0,"ondisk_log_size":7828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260177920,"num_objects":3970,"num_object_clones":0,"num_object_copies":11910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7828,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":315,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324981+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:44:49.324981+0000","last_peered":"2024-09-13T14:44:49.324981+0000","last_clean":"2024-09-13T14:44:49.324981+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:44:49.324981+0000","last_undegraded":"2024-09-13T14:44:49.324981+0000","last_fullsized":"2024-09-13T14:44:49.324981+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":330,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291426+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:44:49.291426+0000","last_peered":"2024-09-13T14:44:49.291426+0000","last_clean":"2024-09-13T14:44:49.291426+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:44:49.291426+0000","last_undegraded":"2024-09-13T14:44:49.291426+0000","last_fullsized":"2024-09-13T14:44:49.291426+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"350'7667","reported_seq":7774,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:04.693547+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:45:04.693547+0000","last_peered":"2024-09-13T14:45:04.693547+0000","last_clean":"2024-09-13T14:45:04.693547+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:45:04.693547+0000","last_undegraded":"2024-09-13T14:45:04.693547+0000","last_fullsized":"2024-09-13T14:45:04.693547+0000","mapping_epoch":303,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7667,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":278,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290305+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:44:49.290305+0000","last_peered":"2024-09-13T14:44:49.290305+0000","last_clean":"2024-09-13T14:44:49.290305+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:44:49.290305+0000","last_undegraded":"2024-09-13T14:44:49.290305+0000","last_fullsized":"2024-09-13T14:44:49.290305+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":595,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328213+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:45:00.328213+0000","last_peered":"2024-09-13T14:45:00.328213+0000","last_clean":"2024-09-13T14:45:00.328213+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:45:00.328213+0000","last_undegraded":"2024-09-13T14:45:00.328213+0000","last_fullsized":"2024-09-13T14:45:00.328213+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"350'7538","reported_seq":7642,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:00.371694+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:45:00.371694+0000","last_peered":"2024-09-13T14:45:00.371694+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:45:00.371694+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7538,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1709}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":461,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324531+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:44:49.324531+0000","last_peered":"2024-09-13T14:44:49.324531+0000","last_clean":"2024-09-13T14:44:49.324531+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:44:49.324531+0000","last_undegraded":"2024-09-13T14:44:49.324531+0000","last_fullsized":"2024-09-13T14:44:49.324531+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":593,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316766+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:45:00.316766+0000","last_peered":"2024-09-13T14:45:00.316766+0000","last_clean":"2024-09-13T14:45:00.316766+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:45:00.316766+0000","last_undegraded":"2024-09-13T14:45:00.316766+0000","last_fullsized":"2024-09-13T14:45:00.316766+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293532+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:44:49.293532+0000","last_peered":"2024-09-13T14:44:49.293532+0000","last_clean":"2024-09-13T14:44:49.293532+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:44:49.293532+0000","last_undegraded":"2024-09-13T14:44:49.293532+0000","last_fullsized":"2024-09-13T14:44:49.293532+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"350'7315","reported_seq":7422,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:04.496413+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:45:04.496413+0000","last_peered":"2024-09-13T14:45:04.496413+0000","last_clean":"2024-09-13T14:45:04.496413+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:45:04.496413+0000","last_undegraded":"2024-09-13T14:45:04.496413+0000","last_fullsized":"2024-09-13T14:45:04.496413+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7315,"log_dups_size":0,"ondisk_log_size":7315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247660544,"num_objects":3779,"num_object_clones":0,"num_object_copies":11337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7315,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":165,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290557+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:44:49.290557+0000","last_peered":"2024-09-13T14:44:49.290557+0000","last_clean":"2024-09-13T14:44:49.290557+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:44:49.290557+0000","last_undegraded":"2024-09-13T14:44:49.290557+0000","last_fullsized":"2024-09-13T14:44:49.290557+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":351,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291300+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:44:49.291300+0000","last_peered":"2024-09-13T14:44:49.291300+0000","last_clean":"2024-09-13T14:44:49.291300+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:44:49.291300+0000","last_undegraded":"2024-09-13T14:44:49.291300+0000","last_fullsized":"2024-09-13T14:44:49.291300+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":63,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293841+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:44:49.293841+0000","last_peered":"2024-09-13T14:44:49.293841+0000","last_clean":"2024-09-13T14:44:49.293841+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:44:49.293841+0000","last_undegraded":"2024-09-13T14:44:49.293841+0000","last_fullsized":"2024-09-13T14:44:49.293841+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"350'7503","reported_seq":12610,"reported_epoch":350,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:04.163092+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:45:04.163092+0000","last_peered":"2024-09-13T14:45:04.163092+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:45:04.163092+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256180224,"num_objects":3909,"num_object_clones":0,"num_object_copies":11727,"num_objects_missing_on_primary":2851,"num_objects_missing":2851,"num_objects_degraded":3226,"num_objects_misplaced":3805,"num_objects_unfound":0,"num_objects_dirty":3909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7503,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1798,"num_bytes_recovered":117833728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2851},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.292657+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:44:49.292657+0000","last_peered":"2024-09-13T14:44:49.292657+0000","last_clean":"2024-09-13T14:44:49.292657+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:44:49.292657+0000","last_undegraded":"2024-09-13T14:44:49.292657+0000","last_fullsized":"2024-09-13T14:44:49.292657+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293935+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:44:49.293935+0000","last_peered":"2024-09-13T14:44:49.293935+0000","last_clean":"2024-09-13T14:44:49.293935+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:44:49.293935+0000","last_undegraded":"2024-09-13T14:44:49.293935+0000","last_fullsized":"2024-09-13T14:44:49.293935+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"350'7406","reported_seq":7513,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:04.656231+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:45:04.656231+0000","last_peered":"2024-09-13T14:45:04.656231+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:45:04.656231+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7406,"log_dups_size":0,"ondisk_log_size":7406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245891072,"num_objects":3752,"num_object_clones":0,"num_object_copies":11256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7406,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1714}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":158,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293192+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:44:49.293192+0000","last_peered":"2024-09-13T14:44:49.293192+0000","last_clean":"2024-09-13T14:44:49.293192+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:44:49.293192+0000","last_undegraded":"2024-09-13T14:44:49.293192+0000","last_fullsized":"2024-09-13T14:44:49.293192+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":585,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.327711+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:45:00.327711+0000","last_peered":"2024-09-13T14:45:00.327711+0000","last_clean":"2024-09-13T14:45:00.327711+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:45:00.327711+0000","last_undegraded":"2024-09-13T14:45:00.327711+0000","last_fullsized":"2024-09-13T14:45:00.327711+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":547,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318602+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:45:00.318602+0000","last_peered":"2024-09-13T14:45:00.318602+0000","last_clean":"2024-09-13T14:45:00.318602+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:45:00.318602+0000","last_undegraded":"2024-09-13T14:45:00.318602+0000","last_fullsized":"2024-09-13T14:45:00.318602+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"350'7427","reported_seq":7489,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:02.691351+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:45:02.691351+0000","last_peered":"2024-09-13T14:45:02.691351+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:45:02.691351+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7427,"log_dups_size":0,"ondisk_log_size":7427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248578048,"num_objects":3793,"num_object_clones":0,"num_object_copies":11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7427,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1703}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":452,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324598+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:44:49.324598+0000","last_peered":"2024-09-13T14:44:49.324598+0000","last_clean":"2024-09-13T14:44:49.324598+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:44:49.324598+0000","last_undegraded":"2024-09-13T14:44:49.324598+0000","last_fullsized":"2024-09-13T14:44:49.324598+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":874,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290681+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:44:49.290681+0000","last_peered":"2024-09-13T14:44:49.290681+0000","last_clean":"2024-09-13T14:44:49.290681+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:44:49.290681+0000","last_undegraded":"2024-09-13T14:44:49.290681+0000","last_fullsized":"2024-09-13T14:44:49.290681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":539,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316263+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:45:00.316263+0000","last_peered":"2024-09-13T14:45:00.316263+0000","last_clean":"2024-09-13T14:45:00.316263+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:45:00.316263+0000","last_undegraded":"2024-09-13T14:45:00.316263+0000","last_fullsized":"2024-09-13T14:45:00.316263+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":316,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.292155+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:44:49.292155+0000","last_peered":"2024-09-13T14:44:49.292155+0000","last_clean":"2024-09-13T14:44:49.292155+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:44:49.292155+0000","last_undegraded":"2024-09-13T14:44:49.292155+0000","last_fullsized":"2024-09-13T14:44:49.292155+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"350'7664","reported_seq":7769,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:04.555745+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:45:04.555745+0000","last_peered":"2024-09-13T14:45:04.555745+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:45:04.555745+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254541824,"num_objects":3884,"num_object_clones":0,"num_object_copies":11652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1796}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":392,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318580+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:45:00.318580+0000","last_peered":"2024-09-13T14:45:00.318580+0000","last_clean":"2024-09-13T14:45:00.318580+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:45:00.318580+0000","last_undegraded":"2024-09-13T14:45:00.318580+0000","last_fullsized":"2024-09-13T14:45:00.318580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":740,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324548+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:44:49.324548+0000","last_peered":"2024-09-13T14:44:49.324548+0000","last_clean":"2024-09-13T14:44:49.324548+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:44:49.324548+0000","last_undegraded":"2024-09-13T14:44:49.324548+0000","last_fullsized":"2024-09-13T14:44:49.324548+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":588,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.324585+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:45:00.324585+0000","last_peered":"2024-09-13T14:45:00.324585+0000","last_clean":"2024-09-13T14:45:00.324585+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:45:00.324585+0000","last_undegraded":"2024-09-13T14:45:00.324585+0000","last_fullsized":"2024-09-13T14:45:00.324585+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":592,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328988+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:45:00.328988+0000","last_peered":"2024-09-13T14:45:00.328988+0000","last_clean":"2024-09-13T14:45:00.328988+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:45:00.328988+0000","last_undegraded":"2024-09-13T14:45:00.328988+0000","last_fullsized":"2024-09-13T14:45:00.328988+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"350'7620","reported_seq":17268,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.395855+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:45:00.395855+0000","last_peered":"2024-09-13T14:45:00.395855+0000","last_clean":"2024-09-13T14:45:00.395855+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:45:00.395855+0000","last_undegraded":"2024-09-13T14:45:00.395855+0000","last_fullsized":"2024-09-13T14:45:00.395855+0000","mapping_epoch":330,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7020,"log_dups_size":0,"ondisk_log_size":7020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265027584,"num_objects":4044,"num_object_clones":0,"num_object_copies":12132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7620,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":481,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324578+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:44:49.324578+0000","last_peered":"2024-09-13T14:44:49.324578+0000","last_clean":"2024-09-13T14:44:49.324578+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:44:49.324578+0000","last_undegraded":"2024-09-13T14:44:49.324578+0000","last_fullsized":"2024-09-13T14:44:49.324578+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":556,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316634+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:45:00.316634+0000","last_peered":"2024-09-13T14:45:00.316634+0000","last_clean":"2024-09-13T14:45:00.316634+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:45:00.316634+0000","last_undegraded":"2024-09-13T14:45:00.316634+0000","last_fullsized":"2024-09-13T14:45:00.316634+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":525,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317791+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:45:00.317791+0000","last_peered":"2024-09-13T14:45:00.317791+0000","last_clean":"2024-09-13T14:45:00.317791+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:45:00.317791+0000","last_undegraded":"2024-09-13T14:45:00.317791+0000","last_fullsized":"2024-09-13T14:45:00.317791+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":99,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290227+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:44:49.290227+0000","last_peered":"2024-09-13T14:44:49.290227+0000","last_clean":"2024-09-13T14:44:49.290227+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:44:49.290227+0000","last_undegraded":"2024-09-13T14:44:49.290227+0000","last_fullsized":"2024-09-13T14:44:49.290227+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"350'7352","reported_seq":7458,"reported_epoch":350,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:00.562679+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:45:00.562679+0000","last_peered":"2024-09-13T14:45:00.562679+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:45:00.562679+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7052,"log_dups_size":0,"ondisk_log_size":7052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254148608,"num_objects":3878,"num_object_clones":0,"num_object_copies":11634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7352,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1631}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":158,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290228+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:44:49.290228+0000","last_peered":"2024-09-13T14:44:49.290228+0000","last_clean":"2024-09-13T14:44:49.290228+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:44:49.290228+0000","last_undegraded":"2024-09-13T14:44:49.290228+0000","last_fullsized":"2024-09-13T14:44:49.290228+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":965,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293295+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:44:49.293295+0000","last_peered":"2024-09-13T14:44:49.293295+0000","last_clean":"2024-09-13T14:44:49.293295+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:44:49.293295+0000","last_undegraded":"2024-09-13T14:44:49.293295+0000","last_fullsized":"2024-09-13T14:44:49.293295+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293631+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:44:49.293631+0000","last_peered":"2024-09-13T14:44:49.293631+0000","last_clean":"2024-09-13T14:44:49.293631+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:44:49.293631+0000","last_undegraded":"2024-09-13T14:44:49.293631+0000","last_fullsized":"2024-09-13T14:44:49.293631+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":545,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318915+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:45:00.318915+0000","last_peered":"2024-09-13T14:45:00.318915+0000","last_clean":"2024-09-13T14:45:00.318915+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:45:00.318915+0000","last_undegraded":"2024-09-13T14:45:00.318915+0000","last_fullsized":"2024-09-13T14:45:00.318915+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"350'7547","reported_seq":16934,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:03.953551+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:45:03.953551+0000","last_peered":"2024-09-13T14:45:03.953551+0000","last_clean":"2024-09-13T14:45:03.953551+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:45:03.953551+0000","last_undegraded":"2024-09-13T14:45:03.953551+0000","last_fullsized":"2024-09-13T14:45:03.953551+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249888768,"num_objects":3813,"num_object_clones":0,"num_object_copies":11439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7547,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":349,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291782+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:44:49.291782+0000","last_peered":"2024-09-13T14:44:49.291782+0000","last_clean":"2024-09-13T14:44:49.291782+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:44:49.291782+0000","last_undegraded":"2024-09-13T14:44:49.291782+0000","last_fullsized":"2024-09-13T14:44:49.291782+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":295,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290310+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:44:49.290310+0000","last_peered":"2024-09-13T14:44:49.290310+0000","last_clean":"2024-09-13T14:44:49.290310+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:44:49.290310+0000","last_undegraded":"2024-09-13T14:44:49.290310+0000","last_fullsized":"2024-09-13T14:44:49.290310+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1124,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317757+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:45:00.317757+0000","last_peered":"2024-09-13T14:45:00.317757+0000","last_clean":"2024-09-13T14:45:00.317757+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:45:00.317757+0000","last_undegraded":"2024-09-13T14:45:00.317757+0000","last_fullsized":"2024-09-13T14:45:00.317757+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":526,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318239+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:45:00.318239+0000","last_peered":"2024-09-13T14:45:00.318239+0000","last_clean":"2024-09-13T14:45:00.318239+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:45:00.318239+0000","last_undegraded":"2024-09-13T14:45:00.318239+0000","last_fullsized":"2024-09-13T14:45:00.318239+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"350'7380","reported_seq":7487,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.392442+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:45:00.392442+0000","last_peered":"2024-09-13T14:45:00.392442+0000","last_clean":"2024-09-13T14:45:00.392442+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:45:00.392442+0000","last_undegraded":"2024-09-13T14:45:00.392442+0000","last_fullsized":"2024-09-13T14:45:00.392442+0000","mapping_epoch":303,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250609664,"num_objects":3824,"num_object_clones":0,"num_object_copies":11472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7380,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":349,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291498+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:44:49.291498+0000","last_peered":"2024-09-13T14:44:49.291498+0000","last_clean":"2024-09-13T14:44:49.291498+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:44:49.291498+0000","last_undegraded":"2024-09-13T14:44:49.291498+0000","last_fullsized":"2024-09-13T14:44:49.291498+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":535,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317587+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:45:00.317587+0000","last_peered":"2024-09-13T14:45:00.317587+0000","last_clean":"2024-09-13T14:45:00.317587+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:45:00.317587+0000","last_undegraded":"2024-09-13T14:45:00.317587+0000","last_fullsized":"2024-09-13T14:45:00.317587+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":744,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.325029+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:44:49.325029+0000","last_peered":"2024-09-13T14:44:49.325029+0000","last_clean":"2024-09-13T14:44:49.325029+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:44:49.325029+0000","last_undegraded":"2024-09-13T14:44:49.325029+0000","last_fullsized":"2024-09-13T14:44:49.325029+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":571,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328119+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:45:00.328119+0000","last_peered":"2024-09-13T14:45:00.328119+0000","last_clean":"2024-09-13T14:45:00.328119+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:45:00.328119+0000","last_undegraded":"2024-09-13T14:45:00.328119+0000","last_fullsized":"2024-09-13T14:45:00.328119+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"350'7785","reported_seq":7892,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:03.436822+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:45:03.436822+0000","last_peered":"2024-09-13T14:45:03.436822+0000","last_clean":"2024-09-13T14:45:03.436822+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:45:03.436822+0000","last_undegraded":"2024-09-13T14:45:03.436822+0000","last_fullsized":"2024-09-13T14:45:03.436822+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7785,"log_dups_size":0,"ondisk_log_size":7785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255918080,"num_objects":3905,"num_object_clones":0,"num_object_copies":11715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7785,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293762+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:44:49.293762+0000","last_peered":"2024-09-13T14:44:49.293762+0000","last_clean":"2024-09-13T14:44:49.293762+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:44:49.293762+0000","last_undegraded":"2024-09-13T14:44:49.293762+0000","last_fullsized":"2024-09-13T14:44:49.293762+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":571,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317333+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:45:00.317333+0000","last_peered":"2024-09-13T14:45:00.317333+0000","last_clean":"2024-09-13T14:45:00.317333+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:45:00.317333+0000","last_undegraded":"2024-09-13T14:45:00.317333+0000","last_fullsized":"2024-09-13T14:45:00.317333+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":911,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317929+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:45:00.317929+0000","last_peered":"2024-09-13T14:45:00.317929+0000","last_clean":"2024-09-13T14:45:00.317929+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:45:00.317929+0000","last_undegraded":"2024-09-13T14:45:00.317929+0000","last_fullsized":"2024-09-13T14:45:00.317929+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":569,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.327545+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:45:00.327545+0000","last_peered":"2024-09-13T14:45:00.327545+0000","last_clean":"2024-09-13T14:45:00.327545+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:45:00.327545+0000","last_undegraded":"2024-09-13T14:45:00.327545+0000","last_fullsized":"2024-09-13T14:45:00.327545+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"350'7675","reported_seq":19036,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:02.695548+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:45:02.695548+0000","last_peered":"2024-09-13T14:45:02.695548+0000","last_clean":"2024-09-13T14:45:02.695548+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:45:02.695548+0000","last_undegraded":"2024-09-13T14:45:02.695548+0000","last_fullsized":"2024-09-13T14:45:02.695548+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7675,"log_dups_size":0,"ondisk_log_size":7675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527192,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7675,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":423,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316940+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:45:00.316940+0000","last_peered":"2024-09-13T14:45:00.316940+0000","last_clean":"2024-09-13T14:45:00.316940+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:45:00.316940+0000","last_undegraded":"2024-09-13T14:45:00.316940+0000","last_fullsized":"2024-09-13T14:45:00.316940+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":606,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328852+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:45:00.328852+0000","last_peered":"2024-09-13T14:45:00.328852+0000","last_clean":"2024-09-13T14:45:00.328852+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:45:00.328852+0000","last_undegraded":"2024-09-13T14:45:00.328852+0000","last_fullsized":"2024-09-13T14:45:00.328852+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":768,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291911+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:44:49.291911+0000","last_peered":"2024-09-13T14:44:49.291911+0000","last_clean":"2024-09-13T14:44:49.291911+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:44:49.291911+0000","last_undegraded":"2024-09-13T14:44:49.291911+0000","last_fullsized":"2024-09-13T14:44:49.291911+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":63,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.292761+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:44:49.292761+0000","last_peered":"2024-09-13T14:44:49.292761+0000","last_clean":"2024-09-13T14:44:49.292761+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:44:49.292761+0000","last_undegraded":"2024-09-13T14:44:49.292761+0000","last_fullsized":"2024-09-13T14:44:49.292761+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293650+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:44:49.293650+0000","last_peered":"2024-09-13T14:44:49.293650+0000","last_clean":"2024-09-13T14:44:49.293650+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:44:49.293650+0000","last_undegraded":"2024-09-13T14:44:49.293650+0000","last_fullsized":"2024-09-13T14:44:49.293650+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":59,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293370+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:44:49.293370+0000","last_peered":"2024-09-13T14:44:49.293370+0000","last_clean":"2024-09-13T14:44:49.293370+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:44:49.293370+0000","last_undegraded":"2024-09-13T14:44:49.293370+0000","last_fullsized":"2024-09-13T14:44:49.293370+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":523,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318085+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:45:00.318085+0000","last_peered":"2024-09-13T14:45:00.318085+0000","last_clean":"2024-09-13T14:45:00.318085+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:45:00.318085+0000","last_undegraded":"2024-09-13T14:45:00.318085+0000","last_fullsized":"2024-09-13T14:45:00.318085+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":349,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.324935+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:44:49.324935+0000","last_peered":"2024-09-13T14:44:49.324935+0000","last_clean":"2024-09-13T14:44:49.324935+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:44:49.324935+0000","last_undegraded":"2024-09-13T14:44:49.324935+0000","last_fullsized":"2024-09-13T14:44:49.324935+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290232+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:44:49.290232+0000","last_peered":"2024-09-13T14:44:49.290232+0000","last_clean":"2024-09-13T14:44:49.290232+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:44:49.290232+0000","last_undegraded":"2024-09-13T14:44:49.290232+0000","last_fullsized":"2024-09-13T14:44:49.290232+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":528,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316423+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:45:00.316423+0000","last_peered":"2024-09-13T14:45:00.316423+0000","last_clean":"2024-09-13T14:45:00.316423+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:45:00.316423+0000","last_undegraded":"2024-09-13T14:45:00.316423+0000","last_fullsized":"2024-09-13T14:45:00.316423+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":579,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328330+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:45:00.328330+0000","last_peered":"2024-09-13T14:45:00.328330+0000","last_clean":"2024-09-13T14:45:00.328330+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:45:00.328330+0000","last_undegraded":"2024-09-13T14:45:00.328330+0000","last_fullsized":"2024-09-13T14:45:00.328330+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":584,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.327500+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:45:00.327500+0000","last_peered":"2024-09-13T14:45:00.327500+0000","last_clean":"2024-09-13T14:45:00.327500+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:45:00.327500+0000","last_undegraded":"2024-09-13T14:45:00.327500+0000","last_fullsized":"2024-09-13T14:45:00.327500+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":347,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.291849+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:44:49.291849+0000","last_peered":"2024-09-13T14:44:49.291849+0000","last_clean":"2024-09-13T14:44:49.291849+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:44:49.291849+0000","last_undegraded":"2024-09-13T14:44:49.291849+0000","last_fullsized":"2024-09-13T14:44:49.291849+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":601,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.327625+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:45:00.327625+0000","last_peered":"2024-09-13T14:45:00.327625+0000","last_clean":"2024-09-13T14:45:00.327625+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:45:00.327625+0000","last_undegraded":"2024-09-13T14:45:00.327625+0000","last_fullsized":"2024-09-13T14:45:00.327625+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":571,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328985+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:45:00.328985+0000","last_peered":"2024-09-13T14:45:00.328985+0000","last_clean":"2024-09-13T14:45:00.328985+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:45:00.328985+0000","last_undegraded":"2024-09-13T14:45:00.328985+0000","last_fullsized":"2024-09-13T14:45:00.328985+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":548,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.318223+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:45:00.318223+0000","last_peered":"2024-09-13T14:45:00.318223+0000","last_clean":"2024-09-13T14:45:00.318223+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:45:00.318223+0000","last_undegraded":"2024-09-13T14:45:00.318223+0000","last_fullsized":"2024-09-13T14:45:00.318223+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":597,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316635+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:45:00.316635+0000","last_peered":"2024-09-13T14:45:00.316635+0000","last_clean":"2024-09-13T14:45:00.316635+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:45:00.316635+0000","last_undegraded":"2024-09-13T14:45:00.316635+0000","last_fullsized":"2024-09-13T14:45:00.316635+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":48,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.317036+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:44:49.317036+0000","last_peered":"2024-09-13T14:44:49.317036+0000","last_clean":"2024-09-13T14:44:49.317036+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:44:49.317036+0000","last_undegraded":"2024-09-13T14:44:49.317036+0000","last_fullsized":"2024-09-13T14:44:49.317036+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":494,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.317926+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:45:00.317926+0000","last_peered":"2024-09-13T14:45:00.317926+0000","last_clean":"2024-09-13T14:45:00.317926+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:45:00.317926+0000","last_undegraded":"2024-09-13T14:45:00.317926+0000","last_fullsized":"2024-09-13T14:45:00.317926+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":575,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.324691+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:45:00.324691+0000","last_peered":"2024-09-13T14:45:00.324691+0000","last_clean":"2024-09-13T14:45:00.324691+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:45:00.324691+0000","last_undegraded":"2024-09-13T14:45:00.324691+0000","last_fullsized":"2024-09-13T14:45:00.324691+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.293583+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:44:49.293583+0000","last_peered":"2024-09-13T14:44:49.293583+0000","last_clean":"2024-09-13T14:44:49.293583+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:44:49.293583+0000","last_undegraded":"2024-09-13T14:44:49.293583+0000","last_fullsized":"2024-09-13T14:44:49.293583+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":315,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290343+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:44:49.290343+0000","last_peered":"2024-09-13T14:44:49.290343+0000","last_clean":"2024-09-13T14:44:49.290343+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:44:49.290343+0000","last_undegraded":"2024-09-13T14:44:49.290343+0000","last_fullsized":"2024-09-13T14:44:49.290343+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.316987+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:44:49.316987+0000","last_peered":"2024-09-13T14:44:49.316987+0000","last_clean":"2024-09-13T14:44:49.316987+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:44:49.316987+0000","last_undegraded":"2024-09-13T14:44:49.316987+0000","last_fullsized":"2024-09-13T14:44:49.316987+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":281,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.290564+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:44:49.290564+0000","last_peered":"2024-09-13T14:44:49.290564+0000","last_clean":"2024-09-13T14:44:49.290564+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:44:49.290564+0000","last_undegraded":"2024-09-13T14:44:49.290564+0000","last_fullsized":"2024-09-13T14:44:49.290564+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":593,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316756+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:45:00.316756+0000","last_peered":"2024-09-13T14:45:00.316756+0000","last_clean":"2024-09-13T14:45:00.316756+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:45:00.316756+0000","last_undegraded":"2024-09-13T14:45:00.316756+0000","last_fullsized":"2024-09-13T14:45:00.316756+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":580,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.316644+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:45:00.316644+0000","last_peered":"2024-09-13T14:45:00.316644+0000","last_clean":"2024-09-13T14:45:00.316644+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:45:00.316644+0000","last_undegraded":"2024-09-13T14:45:00.316644+0000","last_fullsized":"2024-09-13T14:45:00.316644+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":349,"state":"active+clean","last_fresh":"2024-09-13T14:44:49.292451+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:44:49.292451+0000","last_peered":"2024-09-13T14:44:49.292451+0000","last_clean":"2024-09-13T14:44:49.292451+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:44:49.292451+0000","last_undegraded":"2024-09-13T14:44:49.292451+0000","last_fullsized":"2024-09-13T14:44:49.292451+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":585,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-09-13T14:45:00.328627+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:45:00.328627+0000","last_peered":"2024-09-13T14:45:00.328627+0000","last_clean":"2024-09-13T14:45:00.328627+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:45:00.328627+0000","last_undegraded":"2024-09-13T14:45:00.328627+0000","last_fullsized":"2024-09-13T14:45:00.328627+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4057006104,"num_objects":61906,"num_object_clones":0,"num_object_copies":185718,"num_objects_missing_on_primary":2851,"num_objects_missing":2851,"num_objects_degraded":37523,"num_objects_misplaced":3805,"num_objects_unfound":0,"num_objects_dirty":61906,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120772,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23441,"num_bytes_recovered":1475805184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11165052928,"data_stored":11165040712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118672,"ondisk_log_size":118672,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397907,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2187868,"kb_used_data":1296020,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91557284,"statfs":{"total":95995035648,"available":93754658816,"internally_reserved":0,"allocated":1327124480,"data_stored":1325285071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594135,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2186828,"kb_used_data":1596928,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":91558324,"statfs":{"total":95995035648,"available":93755723776,"internally_reserved":0,"allocated":1635254272,"data_stored":1633434617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665589,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":622156,"kb_used_data":297920,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93122996,"statfs":{"total":95995035648,"available":95357947904,"internally_reserved":0,"allocated":305070080,"data_stored":303246756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986591,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777684,"kb_used_data":886156,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":91967468,"statfs":{"total":95995035648,"available":94174687232,"internally_reserved":0,"allocated":907423744,"data_stored":905592433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697150,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2598452,"kb_used_data":1870828,"kb_used_omap":4,"kb_used_meta":727611,"kb_avail":91146700,"statfs":{"total":95995035648,"available":93334220800,"internally_reserved":0,"allocated":1915727872,"data_stored":1913916548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":745073739},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860677,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2410000,"kb_used_data":1784580,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91335152,"statfs":{"total":95995035648,"available":93527195648,"internally_reserved":0,"allocated":1827409920,"data_stored":1825583095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020349,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037184,"kb_used_data":1737784,"kb_used_omap":9,"kb_used_meta":299382,"kb_avail":91707968,"statfs":{"total":95995035648,"available":93908959232,"internally_reserved":0,"allocated":1779490816,"data_stored":1777675118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":306568005},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187739,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2106944,"kb_used_data":1492280,"kb_used_omap":7,"kb_used_meta":614648,"kb_avail":91638208,"statfs":{"total":95995035648,"available":93837524992,"internally_reserved":0,"allocated":1528094720,"data_stored":1526279288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":629399883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1518534656,"data_stored":1518534656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1768095744,"data_stored":1768095744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1819611136,"data_stored":1819607064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1908801536,"data_stored":1908801536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":902168576,"data_stored":902168576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":299438080,"data_stored":299434008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1630470144,"data_stored":1630470144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1317933056,"data_stored":1317928984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:09.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:45:09.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:09.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:09.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:09.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:09.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:09.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:10.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:10.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:10.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:10.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:10.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:10.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:10.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:10 smithi189 ceph-mon[90992]: pgmap v1080: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37453/184629 objects degraded (20.286%); 3805/184629 objects misplaced (2.061%); 793 KiB/s, 12 objects/s recovering 2024-09-13T14:45:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:10.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:10.555 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:10 smithi017 ceph-mon[107556]: pgmap v1080: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37453/184629 objects degraded (20.286%); 3805/184629 objects misplaced (2.061%); 793 KiB/s, 12 objects/s recovering 2024-09-13T14:45:10.556 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:10.556 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:10 smithi017 ceph-mon[103001]: pgmap v1080: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37453/184629 objects degraded (20.286%); 3805/184629 objects misplaced (2.061%); 793 KiB/s, 12 objects/s recovering 2024-09-13T14:45:10.556 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:10.560 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:10.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:10.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:10.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:10.689 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:45:10.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:10.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:10.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:11.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:11.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:11.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:11.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:11.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:11.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:11.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:11.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:11.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:11.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[103001]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:11.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:11 smithi017 ceph-mon[107556]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:45:11.661 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:11.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:11 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2269986567' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:11.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:11 smithi189 ceph-mon[90992]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:45:11.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.809 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:11.911 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:11.931 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:12.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:12.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:12.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:12.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:12.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:12.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:12.455 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:12.455 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:12.456 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:12.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:12.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:12 smithi189 ceph-mon[90992]: pgmap v1082: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37411/183567 objects degraded (20.380%); 3761/183567 objects misplaced (2.049%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:12.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:12.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[103001]: pgmap v1082: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37411/183567 objects degraded (20.380%); 3761/183567 objects misplaced (2.049%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[107556]: pgmap v1082: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 37411/183567 objects degraded (20.380%); 3761/183567 objects misplaced (2.049%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:12.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:12.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:12.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:12.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:13.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:13.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:13.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:13.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:13.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:13.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:45:13.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:13.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:13.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:13.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:45:13.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:13.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:13.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:13.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:13.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:13.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:13.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:14.005 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:14.027 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:14.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:14.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:14.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:45:14.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:14.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:45:14.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:14.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:14.261 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:45:14.261 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:14.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:14.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:45:14.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:14.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:45:14.326 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:14.350 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:45:14.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[103001]: pgmap v1083: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37357/183228 objects degraded (20.388%); 3761/183228 objects misplaced (2.053%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37357/183228 objects degraded (20.388%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1169139826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:14.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:45:14.373 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:14.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:14.397 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:45:14.397 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:14.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:14.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:14.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:14.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:14.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[107556]: pgmap v1083: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37357/183228 objects degraded (20.388%); 3761/183228 objects misplaced (2.053%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37357/183228 objects degraded (20.388%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1169139826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:14 smithi189 ceph-mon[90992]: pgmap v1083: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37357/183228 objects degraded (20.388%); 3761/183228 objects misplaced (2.053%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37357/183228 objects degraded (20.388%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1169139826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:14.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:14.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:14.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:14.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:15.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:15.096 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:15.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:15.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:15.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:15.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:15.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:15.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:15.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:15.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:15.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:15.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:15.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:15.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:15.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:15.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:15.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:16.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:16.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:16.151 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:16.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:16.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:16.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:16.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:16.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:16 smithi017 ceph-mon[103001]: pgmap v1084: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37315/182442 objects degraded (20.453%); 3719/182442 objects misplaced (2.038%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:16.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:16 smithi017 ceph-mon[107556]: pgmap v1084: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37315/182442 objects degraded (20.453%); 3719/182442 objects misplaced (2.038%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:16.505 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:16.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:16.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:16.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:16 smithi189 ceph-mon[90992]: pgmap v1084: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37315/182442 objects degraded (20.453%); 3719/182442 objects misplaced (2.038%); 947 KiB/s, 14 objects/s recovering 2024-09-13T14:45:16.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:16.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:16.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:16.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:17.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:17.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:17.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:17.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:17.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:17.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:17.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:17.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:17] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:45:17.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:17.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:17.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:17.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:17.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:17.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:17.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:17.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:17.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:18.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:18.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:18.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:18.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:18.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:18.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:18.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:18.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:18.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:18.479 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:18 smithi017 ceph-mon[107556]: pgmap v1085: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37235/181899 objects degraded (20.470%); 3719/181899 objects misplaced (2.045%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:18.479 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:18 smithi017 ceph-mon[103001]: pgmap v1085: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37235/181899 objects degraded (20.470%); 3719/181899 objects misplaced (2.045%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:18.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:18.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:18.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:18.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:18 smithi189 ceph-mon[90992]: pgmap v1085: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 37235/181899 objects degraded (20.470%); 3719/181899 objects misplaced (2.045%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:18.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:18.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:18.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:18.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:18.913 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:19.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:19.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:19.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:19.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:19.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:19.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:19.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:19.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:19.411 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37235/181899 objects degraded (20.470%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37235/181899 objects degraded (20.470%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:19.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:19.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:19.555 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:19.555 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 37235/181143 objects degraded (20.556%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1516,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":351,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":60381,"data_bytes":3959917238,"bytes_used":16032677888,"bytes_avail":751927607296,"bytes_total":767960285184,"degraded_objects":37235,"degraded_total":181143,"degraded_ratio":0.20555583158057447,"misplaced_objects":3719,"misplaced_total":181143,"misplaced_ratio":0.02053074090635575,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":1002454,"recovering_keys_per_sec":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":116},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 28s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:45:19.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:19.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:19.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37235/181899 objects degraded (20.470%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:19.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:19.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:19.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:19.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:19.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:20.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:20.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:20.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:20.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:20.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:20.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:20.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:20.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:20.381 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:45:20.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:20.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:20.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:20 smithi189 ceph-mon[90992]: pgmap v1086: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 37235/181143 objects degraded (20.556%); 3719/181143 objects misplaced (2.053%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1835857720' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:20 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3832537140' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:20.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:20.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:20.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:20.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[103001]: pgmap v1086: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 37235/181143 objects degraded (20.556%); 3719/181143 objects misplaced (2.053%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1835857720' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3832537140' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[107556]: pgmap v1086: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 37235/181143 objects degraded (20.556%); 3719/181143 objects misplaced (2.053%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1835857720' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3832537140' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:20.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:20.766 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:45:20.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:20.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:20.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:20.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:21.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:21.039 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:21.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:21.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:21.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:21.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:21.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:21.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:21.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:21.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:21.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:21.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[103001]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:45:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:21 smithi017 ceph-mon[107556]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:45:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:21 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:21 smithi189 ceph-mon[90992]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:45:21.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:21.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:21.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:21.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:22.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:22.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:22.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218864 7.10 7:0bb0242d:::benchmark_data_smithi017_220393_object99730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.38248630800001, 2024-09-13T14:45:22.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1054092660000001, 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218864 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581376+0000", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581377+0000", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.471e-06 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581382+0000", 2024-09-13T14:45:22.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581394+0000", 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1482e-05 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581404+0000", 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0696999999999999e-05 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583298+0000", 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001893175 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583308+0000", 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9830000000000008e-06 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588552+0000", 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.00524463 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588568+0000", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5375000000000001e-05 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.675967+0000", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0873995139999999 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676058+0000", 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1031000000000003e-05 2024-09-13T14:45:22.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686713+0000", 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01065461 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686785+0000", 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2291000000000003e-05 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218870 7.10 7:0fbe9f56:::benchmark_data_smithi017_220393_object99736:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.38244467099997, 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1036150889999998, 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.163 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218870 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581417+0000", 2024-09-13T14:45:22.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581418+0000", 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1099999999999996e-07 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581421+0000", 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9409999999999999e-06 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581428+0000", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2250000000000003e-06 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581434+0000", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.716e-06 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583376+0000", 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001942163 2024-09-13T14:45:22.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583385+0000", 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8380000000000008e-06 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588598+0000", 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0052131259999999 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588606+0000", 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6040000000000004e-06 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676303+0000", 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0876976759999999 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676368+0000", 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5038999999999999e-05 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684968+0000", 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0086000250000000007 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685033+0000", 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4225000000000003e-05 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218875 7.10 7:0fe1bba5:::benchmark_data_smithi017_220393_object99741:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.169 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.38242004199998, 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1039573599999999, 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218875 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581442+0000", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581443+0000", 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2399999999999999e-07 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581445+0000", 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7350000000000001e-06 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581452+0000", 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8709999999999999e-06 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581457+0000", 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1529999999999996e-06 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583857+0000", 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023997739999999999 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583866+0000", 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1630000000000002e-06 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588632+0000", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0047655849999999 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588639+0000", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5739999999999998e-06 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676415+0000", 2024-09-13T14:45:22.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0877759490000001 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.676484+0000", 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9036999999999996e-05 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685350+0000", 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088651589999999992 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685399+0000", 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9935999999999998e-05 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218879 7.10 7:0ecc6a5d:::benchmark_data_smithi017_220393_object99745:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.38239791699999, 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1043238309999999, 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.176 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218879 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581464+0000", 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581465+0000", 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1699999999999999e-07 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581467+0000", 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2759999999999999e-06 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581473+0000", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8640000000000001e-06 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.581478+0000", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.972e-06 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583901+0000", 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024232759999999998 2024-09-13T14:45:22.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.583909+0000", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6909999999999997e-06 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588666+0000", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004757033 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588673+0000", 2024-09-13T14:45:22.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1409999999999999e-06 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.677835+0000", 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.089162218 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.677941+0000", 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105562 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.678107+0000", 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166837 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684704+0000", 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065968249999999997 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685439+0000", 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073457100000000005 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685524+0000", 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5492000000000001e-05 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685575+0000", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0821e-05 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685649+0000", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4110999999999996e-05 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685788+0000", 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013872400000000001 2024-09-13T14:45:22.185 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218930 7.10 7:0b6a92cc:::benchmark_data_smithi017_220393_object99796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.32526145000003, 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0476642279999999, 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.186 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218930 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638601+0000", 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638602+0000", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.263e-06 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638605+0000", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4400000000000001e-06 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638614+0000", 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0650000000000005e-06 2024-09-13T14:45:22.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638620+0000", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5289999999999999e-06 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638718+0000", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7928000000000002e-05 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638727+0000", 2024-09-13T14:45:22.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5519999999999993e-06 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588699+0000", 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94997155300000002 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588706+0000", 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.007e-06 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679498+0000", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.090792016 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679637+0000", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138744 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.679919+0000", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000282081 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.684714+0000", 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047948169999999998 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685830+0000", 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00111637 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685863+0000", 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3269000000000003e-05 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686082+0000", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000218736 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686137+0000", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5451000000000002e-05 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686265+0000", 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012740700000000001 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.193 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218931 7.10 7:084393c0:::benchmark_data_smithi017_220393_object99797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.325233707, 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0480426669999998, 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.194 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218931 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638628+0000", 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638629+0000", 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4999999999999998e-07 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638631+0000", 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0030000000000001e-06 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638637+0000", 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8680000000000004e-06 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.638643+0000", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9059999999999999e-06 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.639140+0000", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049731000000000005 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.639148+0000", 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.276e-06 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588732+0000", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94958390299999995 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588739+0000", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.097e-06 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.682893+0000", 2024-09-13T14:45:22.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.094153873 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.682981+0000", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7678999999999999e-05 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.683143+0000", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016239100000000001 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.685297+0000", 2024-09-13T14:45:22.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021539329999999998 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686379+0000", 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001081568 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686435+0000", 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6307000000000001e-05 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686488+0000", 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2930999999999999e-05 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686544+0000", 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6208000000000001e-05 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.686671+0000", 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127064 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.201 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218937 7.10 7:0ef37e94:::benchmark_data_smithi017_220393_object99803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.32111765600001, 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.059473412, 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218937 2024-09-13T14:45:22.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642744+0000", 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642746+0000", 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.705e-06 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642752+0000", 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6949999999999997e-06 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642768+0000", 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6464999999999999e-05 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642777+0000", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3639999999999999e-06 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642845+0000", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8206000000000004e-05 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.642854+0000", 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4089999999999992e-06 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588765+0000", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94591096799999996 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588772+0000", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0299999999999996e-06 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.687837+0000", 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.099064214 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.687968+0000", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131218 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.688318+0000", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035054800000000001 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698755+0000", 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010436259 2024-09-13T14:45:22.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698865+0000", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011072400000000001 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698919+0000", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3233999999999999e-05 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702047+0000", 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031281019999999998 2024-09-13T14:45:22.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702097+0000", 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0550000000000002e-05 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702218+0000", 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120721 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218940 7.10 7:0c1ab008:::benchmark_data_smithi017_220393_object99806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.31844525399998, 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0815908670000001, 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218940 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645417+0000", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645418+0000", 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4300000000000001e-06 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645424+0000", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4650000000000002e-06 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645436+0000", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2493999999999999e-05 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645444+0000", 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.191e-06 2024-09-13T14:45:22.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645579+0000", 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134559 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.645589+0000", 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009e-05 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588798+0000", 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94320918200000003 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588806+0000", 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.690573+0000", 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1017671490000001 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.690707+0000", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134289 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.691015+0000", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030820199999999999 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702329+0000", 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011313969 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702380+0000", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0553000000000002e-05 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707199+0000", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004819215 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.726803+0000", 2024-09-13T14:45:22.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019604223 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.726859+0000", 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5495999999999999e-05 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727008+0000", 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014901500000000001 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218943 7.10 7:0a0d9885:::benchmark_data_smithi017_220393_object99809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.31786027599998, 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0812243530000001, 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218943 2024-09-13T14:45:22.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646002+0000", 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646003+0000", 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4600000000000003e-07 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646007+0000", 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5120000000000002e-06 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646018+0000", 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1014e-05 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646025+0000", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9990000000000002e-06 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646721+0000", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00069556399999999997 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.646732+0000", 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.065e-05 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588830+0000", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94209882 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588837+0000", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5270000000000004e-06 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693014+0000", 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.104177489 2024-09-13T14:45:22.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693155+0000", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140335 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.693470+0000", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000315357 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702428+0000", 2024-09-13T14:45:22.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089583709999999997 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702480+0000", 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1990999999999997e-05 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707284+0000", 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048038930000000001 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727023+0000", 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019738240000000001 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727070+0000", 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7880000000000002e-05 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727226+0000", 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015576500000000001 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:218979 7.10 7:0cbc6b69:::benchmark_data_smithi017_220393_object99845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.27946867899999, 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0430168480000002, 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 218979 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.226 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684393+0000", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684395+0000", 2024-09-13T14:45:22.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1689999999999999e-06 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684398+0000", 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.692e-06 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684411+0000", 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3132999999999999e-05 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684418+0000", 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7159999999999997e-06 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684474+0000", 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5420000000000001e-05 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.684482+0000", 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6109999999999994e-06 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588860+0000", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90437778700000004 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588867+0000", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8220000000000001e-06 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696172+0000", 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1073055059999999 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696298+0000", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012602499999999999 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.696758+0000", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045934900000000001 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707243+0000", 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010485387000000001 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707325+0000", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2330000000000006e-05 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715552+0000", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082263270000000003 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727236+0000", 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011683901999999999 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727283+0000", 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7753000000000002e-05 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727410+0000", 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126919 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.233 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219048 7.10 7:0d03c00d:::benchmark_data_smithi017_220393_object99914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.21544320800001, 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9802098940000001, 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219048 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748419+0000", 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748420+0000", 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3060000000000001e-06 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748425+0000", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3089999999999997e-06 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748435+0000", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0013999999999999e-05 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748441+0000", 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6379999999999998e-06 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748494+0000", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2748999999999999e-05 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.748503+0000", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8999999999999995e-06 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588908+0000", 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.84040472799999999 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588916+0000", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8210000000000008e-06 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.698952+0000", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1100361009999999 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.699074+0000", 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012147899999999999 2024-09-13T14:45:22.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.699357+0000", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028307199999999999 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707368+0000", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080108569999999997 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707411+0000", 2024-09-13T14:45:22.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3529999999999998e-05 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715711+0000", 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082997000000000001 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.727424+0000", 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011712382 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.728508+0000", 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001084443 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.728629+0000", 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120865 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219076 7.10 7:0e85a74e:::benchmark_data_smithi017_220393_object99942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.241 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.19011022699999, 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9579237699999998, 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219076 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773752+0000", 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773753+0000", 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5409999999999999e-06 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773761+0000", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.52e-06 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773774+0000", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3345e-05 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773782+0000", 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.174e-06 2024-09-13T14:45:22.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773837+0000", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4138000000000002e-05 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.773846+0000", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6290000000000005e-06 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588947+0000", 2024-09-13T14:45:22.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.81510079700000004 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588955+0000", 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2919999999999996e-06 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702527+0000", 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.11357124 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.702670+0000", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014383199999999999 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.703005+0000", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00033464600000000002 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715416+0000", 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012411294 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715596+0000", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017949700000000001 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723195+0000", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075989910000000003 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731482+0000", 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082870700000000005 2024-09-13T14:45:22.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731541+0000", 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8644999999999998e-05 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731676+0000", 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013511900000000001 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219090 7.10 7:095d1507:::benchmark_data_smithi017_220393_object99956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.177182711, 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.945184759, 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219090 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786679+0000", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786681+0000", 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5e-06 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786686+0000", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1089999999999997e-06 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786699+0000", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3156000000000001e-05 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786707+0000", 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4959999999999999e-06 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786773+0000", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6443000000000001e-05 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.786786+0000", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2452999999999999e-05 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588984+0000", 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.80219853399999996 2024-09-13T14:45:22.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.588992+0000", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8860000000000004e-06 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.704886+0000", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.115894175 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.705013+0000", 2024-09-13T14:45:22.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012722100000000001 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.705310+0000", 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029679199999999999 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715507+0000", 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010196828999999999 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715793+0000", 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028566000000000003 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723236+0000", 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074429479999999996 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731685+0000", 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084498100000000003 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731732+0000", 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.689e-05 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731864+0000", 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013185700000000001 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219132 7.10 7:0a96437d:::benchmark_data_smithi017_220393_object99998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.12296913900002, 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8911681979999999, 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219132 2024-09-13T14:45:22.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840893+0000", 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840894+0000", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.15e-06 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840899+0000", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.403e-06 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840912+0000", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2991e-05 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840919+0000", 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1160000000000001e-06 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840978+0000", 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.982e-05 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.840989+0000", 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0308e-05 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589054+0000", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.74806535900000004 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589067+0000", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2627e-05 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707442+0000", 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1183756840000001 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707564+0000", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121487 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.707871+0000", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030680099999999999 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715654+0000", 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007783351 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715913+0000", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025875399999999998 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723276+0000", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073634030000000001 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731874+0000", 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085975170000000007 2024-09-13T14:45:22.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.731928+0000", 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3974000000000002e-05 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732061+0000", 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013345300000000001 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219157 7.10 7:0de9bdb4:::benchmark_data_smithi017_220393_object100023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.08055863499999, 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8489351599999999, 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219157 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883303+0000", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883305+0000", 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1510000000000001e-06 2024-09-13T14:45:22.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883309+0000", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9790000000000004e-06 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883319+0000", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0462000000000001e-05 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883324+0000", 2024-09-13T14:45:22.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3619999999999997e-06 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883370+0000", 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5890999999999998e-05 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.883377+0000", 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3110000000000002e-06 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589109+0000", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.70573227299999997 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589118+0000", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6109999999999994e-06 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.709675+0000", 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1205571809999999 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.709796+0000", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001208 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.710116+0000", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032086300000000001 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715850+0000", 2024-09-13T14:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057340749999999999 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723068+0000", 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072171129999999998 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723317+0000", 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024899399999999999 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732070+0000", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087533660000000003 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732115+0000", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5197000000000001e-05 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732239+0000", 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012353099999999999 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.273 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45566.0:219172 7.10 7:0f8dfd6f:::benchmark_data_smithi017_220393_object100038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 390.04018834700003, 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8087410400000001, 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.275 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45566", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 219172 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923674+0000", 2024-09-13T14:45:22.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923675+0000", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0139999999999999e-06 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923679+0000", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3699999999999997e-06 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923690+0000", 2024-09-13T14:45:22.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0795000000000001e-05 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923695+0000", 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4299999999999997e-06 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923746+0000", 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0368000000000003e-05 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:51.923754+0000", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0320000000000003e-06 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589297+0000", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.66554344600000004 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:52.589307+0000", 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0094e-05 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.712072+0000", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.122764936 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.712972+0000", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00089972299999999995 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,5", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.713337+0000", 2024-09-13T14:45:22.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036460799999999999 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.715969+0000", 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026328160000000001 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723112+0000", 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071422380000000004 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.723357+0000", 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024494000000000002 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732247+0000", 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088901480000000005 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732291+0000", 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3951e-05 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:38:53.732415+0000", 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012413100000000001 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89668 8.0 8:026d1a37:::benchmark_data_smithi017_253885_object89667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.681730+0000", 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 278.28213257099998, 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4706326320000001, 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.283 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89668 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681730+0000", 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681730+0000", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681736+0000", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2999999999999998e-06 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681761+0000", 2024-09-13T14:45:22.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5091000000000001e-05 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681779+0000", 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7878000000000001e-05 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681788+0000", 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8380000000000008e-06 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.681848+0000", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0748999999999997e-05 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.682207+0000", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035809299999999997 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.682688+0000", 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000481903 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.699493+0000", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016804515999999999 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711733+0000", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012240061999999999 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711747+0000", 2024-09-13T14:45:22.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.428e-05 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691134+0000", 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97938712999999999 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691151+0000", 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6880000000000001e-05 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152253+0000", 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4611017829999999 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152362+0000", 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109129 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89672 8.0 8:027de026:::benchmark_data_smithi017_253885_object89671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.687738+0000", 2024-09-13T14:45:22.290 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 278.27612424400002, 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.46475015, 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89672 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687738+0000", 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687738+0000", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687740+0000", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.001e-06 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687764+0000", 2024-09-13T14:45:22.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4566000000000002e-05 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687782+0000", 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8051999999999999e-05 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687792+0000", 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0440000000000001e-06 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687900+0000", 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108903 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.688059+0000", 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015847300000000001 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1", 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.688209+0000", 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015017900000000001 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.699499+0000", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01128993 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711777+0000", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012277913999999999 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711782+0000", 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.938e-06 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691203+0000", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97942093399999997 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691212+0000", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5049999999999996e-06 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152398+0000", 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.461186103 2024-09-13T14:45:22.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152488+0000", 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9608e-05 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89673 8.0 8:0f9d784c:::benchmark_data_smithi017_253885_object89672:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.687813+0000", 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 278.276048715, 2024-09-13T14:45:22.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4648922510000002, 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89673 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687813+0000", 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687813+0000", 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687814+0000", 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0100000000000005e-07 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687835+0000", 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0591999999999999e-05 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687842+0000", 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4900000000000003e-06 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.687847+0000", 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4670000000000003e-06 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.691387+0000", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003539361 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.691524+0000", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137181 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.691668+0000", 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000143719 2024-09-13T14:45:22.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.702260+0000", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010591929 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711802+0000", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095425830000000003 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.711807+0000", 2024-09-13T14:45:22.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5079999999999999e-06 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691314+0000", 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97950771800000003 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691324+0000", 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7000000000000003e-06 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152523+0000", 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.461198408 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.152706+0000", 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018309399999999999 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89695 8.8 8:159722ab:::benchmark_data_smithi017_253885_object89694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:44.691721+0000", 2024-09-13T14:45:22.304 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 277.27214113500003, 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.487292525, 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89695 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691721+0000", 2024-09-13T14:45:22.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691721+0000", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691757+0000", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.561e-05 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691783+0000", 2024-09-13T14:45:22.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6427999999999998e-05 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691794+0000", 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0560000000000001e-05 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691800+0000", 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8870000000000003e-06 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691858+0000", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7250000000000002e-05 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.691866+0000", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8979999999999996e-06 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.149928+0000", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4580624150000001 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:45:22.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.150030+0000", 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102249 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.150137+0000", 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106585 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.155813+0000", 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056759059999999997 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.162677+0000", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068646369999999998 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.176903+0000", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014225191999999999 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.176974+0000", 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1583999999999998e-05 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:45:22.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.179014+0000", 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020393239999999999 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:45:22.311 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:45:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:22.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:22.413 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:22.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:22.536 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:22.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:22.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:22.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:22 smithi017 ceph-mon[103001]: pgmap v1088: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37191/180090 objects degraded (20.651%); 3675/180090 objects misplaced (2.041%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:22 smithi017 ceph-mon[107556]: pgmap v1088: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37191/180090 objects degraded (20.651%); 3675/180090 objects misplaced (2.041%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:22 smithi189 ceph-mon[90992]: pgmap v1088: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 37191/180090 objects degraded (20.651%); 3675/180090 objects misplaced (2.041%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:45:22.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:22.813 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:22.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:22.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:23.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:23.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:23.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:23.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:23.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:23.314 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:23.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:23.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:23.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:23.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:23.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:45:23.648 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:23.651 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:23.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:23.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:23.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:23.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:23.869 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:23.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:23.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:23.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:24.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:24.068 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:24.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:24.192 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:24.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:24.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:24.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:24.443 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:24 smithi017 ceph-mon[107556]: pgmap v1089: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37125/179679 objects degraded (20.662%); 3675/179679 objects misplaced (2.045%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:45:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 37125/179679 objects degraded (20.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:24 smithi017 ceph-mon[103001]: pgmap v1089: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37125/179679 objects degraded (20.662%); 3675/179679 objects misplaced (2.045%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:45:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 37125/179679 objects degraded (20.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:24.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:24.561 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:24 smithi189 ceph-mon[90992]: pgmap v1089: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 37125/179679 objects degraded (20.662%); 3675/179679 objects misplaced (2.045%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:45:24.561 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 37125/179679 objects degraded (20.662%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:24.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:24.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:24.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:24.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:24.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:24.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:24.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:24.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:25.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:25.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:25.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:25.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:25.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:25.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:25.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:25.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:25.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:25.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:25.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:25.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:25.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:45:25.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:25.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:25.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:25.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:25.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:25.987 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123541 8.2 8:4bc0ea55:::benchmark_data_smithi017_253885_object32200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.018487+0000", 2024-09-13T14:45:26.007 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.98562896600000005, 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98565340300000004, 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123541 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018487+0000", 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018487+0000", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018486+0000", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018492+0000", 2024-09-13T14:45:26.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8069999999999996e-06 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018502+0000", 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0298e-05 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.018507+0000", 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8029999999999996e-06 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.119116+0000", 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.10060949800000001 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.124597+0000", 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0054809979999999999 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.189933+0000", 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.065335357999999996 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.195608+0000", 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0056754259999999999 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123360 8.2 8:42ce6b96:::benchmark_data_smithi017_253885_object32019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.441037+0000", 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.563078467, 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5631538780000001, 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.012 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123360 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441037+0000", 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441037+0000", 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441036+0000", 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441044+0000", 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0190000000000006e-06 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441056+0000", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1868000000000001e-05 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.441062+0000", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7239999999999997e-06 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.540216+0000", 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.099154375000000003 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.548024+0000", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078083349999999996 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.608312+0000", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.060287441999999997 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.615118+0000", 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068062050000000001 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.016 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123545 8.2 8:4e7c5f0c:::benchmark_data_smithi017_253885_object32204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.032424+0000", 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.97169160799999998, 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97181942600000004, 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123545 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.017 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032424+0000", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032424+0000", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032423+0000", 2024-09-13T14:45:26.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032430+0000", 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8340000000000001e-06 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032441+0000", 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0888e-05 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.032445+0000", 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7090000000000001e-06 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.246972+0000", 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.214526353 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.252484+0000", 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055125 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.281438+0000", 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.028954119 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.287399+0000", 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059603939999999999 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.021 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123546 8.2 8:44f87ec6:::benchmark_data_smithi017_253885_object32205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.233930+0000", 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.77018630799999999, 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.77035791099999995, 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123546 2024-09-13T14:45:26.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233930+0000", 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233930+0000", 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.023 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233928+0000", 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233935+0000", 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5520000000000001e-06 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233948+0000", 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2882e-05 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.233953+0000", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7969999999999997e-06 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.254717+0000", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020763462999999999 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.259982+0000", 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0052654490000000002 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.308161+0000", 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048178701999999997 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.314195+0000", 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0060343030000000004 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.026 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123514 8.2 8:49744802:::benchmark_data_smithi017_253885_object32173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.898440+0000", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.1056755069999999, 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1058941959999999, 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123514 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898440+0000", 2024-09-13T14:45:26.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.028 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898440+0000", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898439+0000", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898446+0000", 2024-09-13T14:45:26.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0400000000000004e-06 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898458+0000", 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1882000000000001e-05 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.898463+0000", 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5729999999999998e-06 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.030 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.010508+0000", 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.112044797 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.016254+0000", 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057460209999999996 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.074059+0000", 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.057804530999999999 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.080072+0000", 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0060135290000000001 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123517 8.2 8:455e2c6a:::benchmark_data_smithi017_253885_object32176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.908267+0000", 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.0958487539999999, 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.096110602, 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.032 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123517 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908267+0000", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908267+0000", 2024-09-13T14:45:26.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908265+0000", 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908273+0000", 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6170000000000001e-06 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908288+0000", 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4487e-05 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.908295+0000", 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8839999999999994e-06 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.016945+0000", 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.108649914 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.022609+0000", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005663174 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.102050+0000", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079441538000000006 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.107932+0000", 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058818189999999999 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.036 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123589 8.2 8:438d1932:::benchmark_data_smithi017_253885_object32248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.862423+0000", 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.141692504, 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14199690600000001, 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.037 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123589 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862423+0000", 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862423+0000", 2024-09-13T14:45:26.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862421+0000", 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862434+0000", 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2452999999999999e-05 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862451+0000", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7234000000000001e-05 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.862462+0000", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.102e-05 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.896568+0000", 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.034105797 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.902771+0000", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062034660000000004 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.990497+0000", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087726344999999997 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.996391+0000", 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058937249999999998 2024-09-13T14:45:26.041 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123338 8.2 8:4d7ebe15:::benchmark_data_smithi017_253885_object31997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.253839+0000", 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.750277072, 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.750622294, 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.042 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123338 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253839+0000", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253839+0000", 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253837+0000", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253845+0000", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2679999999999997e-06 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253856+0000", 2024-09-13T14:45:26.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1141999999999999e-05 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.253860+0000", 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6079999999999998e-06 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.318651+0000", 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.064791011999999995 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.323938+0000", 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0052863789999999999 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.507077+0000", 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18313943199999999 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.513274+0000", 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061964869999999997 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.936656+0000", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.423381719 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.943770+0000", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007114932 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.956505+0000", 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012734550000000001 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.996928+0000", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040422901999999997 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.998593+0000", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0016654059999999999 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:26.000388+0000", 2024-09-13T14:45:26.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0017950869999999999 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:26.003121+0000", 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027322990000000001 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123548 8.2 8:4dc5afef:::benchmark_data_smithi017_253885_object32207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.049 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.486076+0000", 2024-09-13T14:45:26.050 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.51803974900000005, 2024-09-13T14:45:26.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51844679400000004, 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123548 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486076+0000", 2024-09-13T14:45:26.051 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486076+0000", 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486075+0000", 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486084+0000", 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0180000000000007e-06 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486099+0000", 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5588999999999999e-05 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.486106+0000", 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1860000000000003e-06 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.538939+0000", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052832954000000001 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.545522+0000", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0065822210000000001 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.586636+0000", 2024-09-13T14:45:26.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.041114691000000002 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.592599+0000", 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059625659999999999 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123592 8.2 8:4e49717f:::benchmark_data_smithi017_253885_object32251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.878234+0000", 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.125882035, 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12632974399999999, 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for missing object", 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123592 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.056 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878234+0000", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878234+0000", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878233+0000", 2024-09-13T14:45:26.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878240+0000", 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9090000000000003e-06 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878251+0000", 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1701e-05 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.878256+0000", 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7639999999999996e-06 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.903808+0000", 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025552353 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.911696+0000", 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078880770000000003 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.059 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123549 8.2 8:4f2704ba:::benchmark_data_smithi017_253885_object32208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.505957+0000", 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.498158455, 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.498641416, 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123549 2024-09-13T14:45:26.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505957+0000", 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505957+0000", 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.061 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505956+0000", 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505966+0000", 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9299999999999998e-06 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505980+0000", 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4348000000000001e-05 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.505989+0000", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.268e-06 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.546101+0000", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040111586999999997 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.551635+0000", 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055341080000000003 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.662165+0000", 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.110529616 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.668023+0000", 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005858551 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.064 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123593 8.2 8.ca54eab2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.879828+0000", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.124287966, 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.12481038, 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123593 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.065 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879828+0000", 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879828+0000", 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879827+0000", 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.066 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879833+0000", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.429e-06 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879845+0000", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2439999999999999e-05 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.879849+0000", 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2200000000000003e-06 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.067 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123410 8.2 8:4d82689e:::benchmark_data_smithi017_253885_object32069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.589158+0000", 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.414957917, 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.415514833, 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123410 2024-09-13T14:45:26.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589158+0000", 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589158+0000", 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589156+0000", 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589164+0000", 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0839999999999992e-06 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589180+0000", 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5179e-05 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.589186+0000", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2539999999999998e-06 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.671703+0000", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082517413999999997 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.678157+0000", 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064539410000000004 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.733276+0000", 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.055119179999999997 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.742714+0000", 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0094379709999999999 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123467 8.2 8:4e8d7a77:::benchmark_data_smithi017_253885_object32126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.072 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:24.768140+0000", 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.235975866, 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2365734580000001, 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123467 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768140+0000", 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768140+0000", 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768139+0000", 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768146+0000", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3529999999999997e-06 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768159+0000", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2695e-05 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:24.768164+0000", 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8289999999999998e-06 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.000939+0000", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.232775592 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.009555+0000", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0086157200000000003 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.044649+0000", 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035094338000000003 2024-09-13T14:45:26.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.051221+0000", 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0065716000000000004 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "MOSDPGPush(8.2 352/317 [PushOp(8:4e49717f:::benchmark_data_smithi017_253885_object32251:head, version: 307'1999, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:4e49717f:::benchmark_data_smithi017_253885_object32251:head@307'1999, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.912109+0000", 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.092006770000000002, 2024-09-13T14:45:26.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.092664910000000003, 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912109+0000", 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912109+0000", 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912108+0000", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912181+0000", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2151999999999996e-05 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912195+0000", 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4578e-05 2024-09-13T14:45:26.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.912206+0000", 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0648999999999999e-05 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.35952.0:123338 8.2 e352/317 ondisk, result = 0)", 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.959964+0000", 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.044152010999999998, 2024-09-13T14:45:26.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.04483467, 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959964+0000", 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959964+0000", 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959963+0000", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959972+0000", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8269999999999993e-06 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959987+0000", 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4453e-05 2024-09-13T14:45:26.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.959994+0000", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8569999999999997e-06 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.999304+0000", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039310327999999999 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.999860+0000", 2024-09-13T14:45:26.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00055587200000000001 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123537 8.2 8:4ad32bb3:::benchmark_data_smithi017_253885_object32196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.007936+0000", 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.99617995599999998, 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99690641899999999, 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123537 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007936+0000", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007936+0000", 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007934+0000", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007942+0000", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3660000000000003e-06 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007955+0000", 2024-09-13T14:45:26.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3594999999999999e-05 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.007963+0000", 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9500000000000001e-06 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.113008+0000", 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.105044393 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.118535+0000", 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005527232 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.221498+0000", 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.10296345899999999 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.227216+0000", 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005717761 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:123559 8.2 8:4237785b:::benchmark_data_smithi017_253885_object32218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:25.579840+0000", 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.42427579999999998, 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.425044743, 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for rw locks", 2024-09-13T14:45:26.089 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123559 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579840+0000", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579840+0000", 2024-09-13T14:45:26.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579839+0000", 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579847+0000", 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.884e-06 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579861+0000", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.377e-05 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.579867+0000", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0079999999999999e-06 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.689862+0000", 2024-09-13T14:45:26.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.109994618 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.696519+0000", 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066573140000000001 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.731683+0000", 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035163959000000002 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:26.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:25.737926+0000", 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006243012 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 18 2024-09-13T14:45:26.094 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:45:26.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:26.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:26.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:26.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:26.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:26.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.369 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:26.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:26.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:26.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:26.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:26.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:26.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:26 smithi017 ceph-mon[103001]: pgmap v1090: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37081/178671 objects degraded (20.754%); 3631/178671 objects misplaced (2.032%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:26 smithi017 ceph-mon[107556]: pgmap v1090: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37081/178671 objects degraded (20.754%); 3631/178671 objects misplaced (2.032%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:26.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:26.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:26 smithi189 ceph-mon[90992]: pgmap v1090: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 37081/178671 objects degraded (20.754%); 3631/178671 objects misplaced (2.032%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:26.831 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:45:26.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:26.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:27.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:27.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:27.123 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:27.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:27.148 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:27.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:27.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:27.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:27.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:27] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-13T14:45:27.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:27.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:27.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:27.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:27.651 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:27.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:27.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:27.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:27.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:27.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:28.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:28.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:28.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:28.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:28.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:28.257 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:28.257 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[103001]: pgmap v1091: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177903 objects degraded (20.789%); 3631/177903 objects misplaced (2.041%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:28.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:28.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:28.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:28 smithi017 ceph-mon[107556]: pgmap v1091: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177903 objects degraded (20.789%); 3631/177903 objects misplaced (2.041%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:28.258 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:45:28.259 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:28.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:28.278 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:45:28.278 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:28.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:28.297 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:45:28.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:28.316 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:45:28.317 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:28.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:45:28.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:28.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:45:28.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:28.365 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:45:28.365 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:28.382 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:45:28.382 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:28.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:28.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:28.413 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:28.413 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:28.413 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:28 smithi189 ceph-mon[90992]: pgmap v1091: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177903 objects degraded (20.789%); 3631/177903 objects misplaced (2.041%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:28.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:28.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:28.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:28.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:28.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:28.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:28.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:28.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:29.009 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:29.010 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:29.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:29.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:29.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:29.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:29.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3642821222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36985/177903 objects degraded (20.789%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:29.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:29.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:29.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3642821222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:29.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36985/177903 objects degraded (20.789%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:29.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3642821222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:29.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36985/177903 objects degraded (20.789%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:29.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:29.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:29.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:29.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:29.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:29.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:29.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:29.792 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:29.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:29.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:45:29.916 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:29.951 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:30.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:30.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:30.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:30.163 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:45:30.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:30.210 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:30.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:30.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:30.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:30 smithi189 ceph-mon[90992]: pgmap v1092: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177132 objects degraded (20.880%); 3631/177132 objects misplaced (2.050%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:30.383 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:30.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:30.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:30.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:30 smithi017 ceph-mon[103001]: pgmap v1092: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177132 objects degraded (20.880%); 3631/177132 objects misplaced (2.050%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:30 smithi017 ceph-mon[107556]: pgmap v1092: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 36985/177132 objects degraded (20.880%); 3631/177132 objects misplaced (2.050%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:30.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:30.543 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:30.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:30.669 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:30.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:30.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:30.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:30.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:31.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:31.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:31.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:31.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:31.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:31.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:31.298 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:31.306 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:31 smithi189 ceph-mon[90992]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:31.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:31 smithi017 ceph-mon[103001]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:31.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:31 smithi017 ceph-mon[107556]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:31.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:31.410 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1092,"stamp":"2024-09-13T14:45:29.224207+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3872295606,"num_objects":59044,"num_object_clones":0,"num_object_copies":177132,"num_objects_missing_on_primary":2679,"num_objects_missing":2679,"num_objects_degraded":36985,"num_objects_misplaced":3631,"num_objects_unfound":0,"num_objects_dirty":59044,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":124609,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23855,"num_bytes_recovered":1499367581,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122194,"ondisk_log_size":122194,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15460944,"kb_used_data":10471376,"kb_used_omap":44,"kb_used_meta":4989459,"kb_avail":734500272,"statfs":{"total":767960285184,"available":752128278528,"internally_reserved":0,"allocated":10722689024,"data_stored":10707972647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5109206263},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,0,0,3,0,1,2,6,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-87621632,"num_objects":-1337,"num_object_clones":0,"num_object_copies":-4011,"num_objects_missing_on_primary":-88,"num_objects_missing":-88,"num_objects_degraded":-250,"num_objects_misplaced":-88,"num_objects_unfound":0,"num_objects_dirty":-1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1337,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":169,"num_bytes_recovered":11075584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002576"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":588,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.405368+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:45:20.405368+0000","last_peered":"2024-09-13T14:45:20.405368+0000","last_clean":"2024-09-13T14:45:20.405368+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:45:20.405368+0000","last_undegraded":"2024-09-13T14:45:20.405368+0000","last_fullsized":"2024-09-13T14:45:20.405368+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":350,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522743+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:45:10.522743+0000","last_peered":"2024-09-13T14:45:10.522743+0000","last_clean":"2024-09-13T14:45:10.522743+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:45:10.522743+0000","last_undegraded":"2024-09-13T14:45:10.522743+0000","last_fullsized":"2024-09-13T14:45:10.522743+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344469+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:45:10.344469+0000","last_peered":"2024-09-13T14:45:10.344469+0000","last_clean":"2024-09-13T14:45:10.344469+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:45:10.344469+0000","last_undegraded":"2024-09-13T14:45:10.344469+0000","last_fullsized":"2024-09-13T14:45:10.344469+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":573,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.392395+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:45:20.392395+0000","last_peered":"2024-09-13T14:45:20.392395+0000","last_clean":"2024-09-13T14:45:20.392395+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:45:20.392395+0000","last_undegraded":"2024-09-13T14:45:20.392395+0000","last_fullsized":"2024-09-13T14:45:20.392395+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":573,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390497+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:45:20.390497+0000","last_peered":"2024-09-13T14:45:20.390497+0000","last_clean":"2024-09-13T14:45:20.390497+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:45:20.390497+0000","last_undegraded":"2024-09-13T14:45:20.390497+0000","last_fullsized":"2024-09-13T14:45:20.390497+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":319,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522740+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:45:10.522740+0000","last_peered":"2024-09-13T14:45:10.522740+0000","last_clean":"2024-09-13T14:45:10.522740+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:45:10.522740+0000","last_undegraded":"2024-09-13T14:45:10.522740+0000","last_fullsized":"2024-09-13T14:45:10.522740+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":587,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.405143+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:45:20.405143+0000","last_peered":"2024-09-13T14:45:20.405143+0000","last_clean":"2024-09-13T14:45:20.405143+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:45:20.405143+0000","last_undegraded":"2024-09-13T14:45:20.405143+0000","last_fullsized":"2024-09-13T14:45:20.405143+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":341,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.343961+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:45:10.343961+0000","last_peered":"2024-09-13T14:45:10.343961+0000","last_clean":"2024-09-13T14:45:10.343961+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:45:10.343961+0000","last_undegraded":"2024-09-13T14:45:10.343961+0000","last_fullsized":"2024-09-13T14:45:10.343961+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"352'7674","reported_seq":13866,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:26.910092+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:45:26.910092+0000","last_peered":"2024-09-13T14:45:26.910092+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:45:26.910092+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7674,"log_dups_size":0,"ondisk_log_size":7674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6147,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7674,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2029,"num_bytes_recovered":132972544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2632}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":345,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522989+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:45:10.522989+0000","last_peered":"2024-09-13T14:45:10.522989+0000","last_clean":"2024-09-13T14:45:10.522989+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:45:10.522989+0000","last_undegraded":"2024-09-13T14:45:10.522989+0000","last_fullsized":"2024-09-13T14:45:10.522989+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":264,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344045+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:45:10.344045+0000","last_peered":"2024-09-13T14:45:10.344045+0000","last_clean":"2024-09-13T14:45:10.344045+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:45:10.344045+0000","last_undegraded":"2024-09-13T14:45:10.344045+0000","last_fullsized":"2024-09-13T14:45:10.344045+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"352'7758","reported_seq":17307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:27.778506+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:45:27.778506+0000","last_peered":"2024-09-13T14:45:27.778506+0000","last_clean":"2024-09-13T14:45:27.778506+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:45:27.778506+0000","last_undegraded":"2024-09-13T14:45:27.778506+0000","last_fullsized":"2024-09-13T14:45:27.778506+0000","mapping_epoch":326,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244449280,"num_objects":3730,"num_object_clones":0,"num_object_copies":11190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":363,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.523031+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:45:10.523031+0000","last_peered":"2024-09-13T14:45:10.523031+0000","last_clean":"2024-09-13T14:45:10.523031+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:45:10.523031+0000","last_undegraded":"2024-09-13T14:45:10.523031+0000","last_fullsized":"2024-09-13T14:45:10.523031+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387964+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:45:10.387964+0000","last_peered":"2024-09-13T14:45:10.387964+0000","last_clean":"2024-09-13T14:45:10.387964+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:45:10.387964+0000","last_undegraded":"2024-09-13T14:45:10.387964+0000","last_fullsized":"2024-09-13T14:45:10.387964+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"352'8055","reported_seq":8166,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:28.683972+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:45:28.683972+0000","last_peered":"2024-09-13T14:45:28.683972+0000","last_clean":"2024-09-13T14:45:28.683972+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:45:28.683972+0000","last_undegraded":"2024-09-13T14:45:28.683972+0000","last_fullsized":"2024-09-13T14:45:28.683972+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8055,"log_dups_size":0,"ondisk_log_size":8055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8055,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.523002+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:45:10.523002+0000","last_peered":"2024-09-13T14:45:10.523002+0000","last_clean":"2024-09-13T14:45:10.523002+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:45:10.523002+0000","last_undegraded":"2024-09-13T14:45:10.523002+0000","last_fullsized":"2024-09-13T14:45:10.523002+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":332,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344009+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:45:10.344009+0000","last_peered":"2024-09-13T14:45:10.344009+0000","last_clean":"2024-09-13T14:45:10.344009+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:45:10.344009+0000","last_undegraded":"2024-09-13T14:45:10.344009+0000","last_fullsized":"2024-09-13T14:45:10.344009+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"352'7818","reported_seq":7929,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:24.777083+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:45:24.777083+0000","last_peered":"2024-09-13T14:45:24.777083+0000","last_clean":"2024-09-13T14:45:24.777083+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:45:24.777083+0000","last_undegraded":"2024-09-13T14:45:24.777083+0000","last_fullsized":"2024-09-13T14:45:24.777083+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244318208,"num_objects":3728,"num_object_clones":0,"num_object_copies":11184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7818,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":280,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344781+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:45:10.344781+0000","last_peered":"2024-09-13T14:45:10.344781+0000","last_clean":"2024-09-13T14:45:10.344781+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:45:10.344781+0000","last_undegraded":"2024-09-13T14:45:10.344781+0000","last_fullsized":"2024-09-13T14:45:10.344781+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":599,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390255+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:45:20.390255+0000","last_peered":"2024-09-13T14:45:20.390255+0000","last_clean":"2024-09-13T14:45:20.390255+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:45:20.390255+0000","last_undegraded":"2024-09-13T14:45:20.390255+0000","last_fullsized":"2024-09-13T14:45:20.390255+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"352'7738","reported_seq":7846,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:24.860328+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:45:24.860328+0000","last_peered":"2024-09-13T14:45:24.860328+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:45:24.860328+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244056064,"num_objects":3724,"num_object_clones":0,"num_object_copies":11172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7738,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1909}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":463,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522542+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:45:10.522542+0000","last_peered":"2024-09-13T14:45:10.522542+0000","last_clean":"2024-09-13T14:45:10.522542+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:45:10.522542+0000","last_undegraded":"2024-09-13T14:45:10.522542+0000","last_fullsized":"2024-09-13T14:45:10.522542+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":597,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.405040+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:45:20.405040+0000","last_peered":"2024-09-13T14:45:20.405040+0000","last_clean":"2024-09-13T14:45:20.405040+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:45:20.405040+0000","last_undegraded":"2024-09-13T14:45:20.405040+0000","last_fullsized":"2024-09-13T14:45:20.405040+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388281+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:45:10.388281+0000","last_peered":"2024-09-13T14:45:10.388281+0000","last_clean":"2024-09-13T14:45:10.388281+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:45:10.388281+0000","last_undegraded":"2024-09-13T14:45:10.388281+0000","last_fullsized":"2024-09-13T14:45:10.388281+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"352'7473","reported_seq":7584,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:24.977290+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:45:24.977290+0000","last_peered":"2024-09-13T14:45:24.977290+0000","last_clean":"2024-09-13T14:45:24.977290+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:45:24.977290+0000","last_undegraded":"2024-09-13T14:45:24.977290+0000","last_fullsized":"2024-09-13T14:45:24.977290+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7473,"log_dups_size":0,"ondisk_log_size":7473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7473,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":167,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344121+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:45:10.344121+0000","last_peered":"2024-09-13T14:45:10.344121+0000","last_clean":"2024-09-13T14:45:10.344121+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:45:10.344121+0000","last_undegraded":"2024-09-13T14:45:10.344121+0000","last_fullsized":"2024-09-13T14:45:10.344121+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.343873+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:45:10.343873+0000","last_peered":"2024-09-13T14:45:10.343873+0000","last_clean":"2024-09-13T14:45:10.343873+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:45:10.343873+0000","last_undegraded":"2024-09-13T14:45:10.343873+0000","last_fullsized":"2024-09-13T14:45:10.343873+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":65,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388383+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:45:10.388383+0000","last_peered":"2024-09-13T14:45:10.388383+0000","last_clean":"2024-09-13T14:45:10.388383+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:45:10.388383+0000","last_undegraded":"2024-09-13T14:45:10.388383+0000","last_fullsized":"2024-09-13T14:45:10.388383+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"352'7678","reported_seq":13305,"reported_epoch":352,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:24.337885+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:45:24.337885+0000","last_peered":"2024-09-13T14:45:24.337885+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:45:24.337885+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7378,"log_dups_size":0,"ondisk_log_size":7378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244711424,"num_objects":3734,"num_object_clones":0,"num_object_copies":11202,"num_objects_missing_on_primary":2679,"num_objects_missing":2679,"num_objects_degraded":3054,"num_objects_misplaced":3631,"num_objects_unfound":0,"num_objects_dirty":3734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7678,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1972,"num_bytes_recovered":129236992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2679},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387779+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:45:10.387779+0000","last_peered":"2024-09-13T14:45:10.387779+0000","last_clean":"2024-09-13T14:45:10.387779+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:45:10.387779+0000","last_undegraded":"2024-09-13T14:45:10.387779+0000","last_fullsized":"2024-09-13T14:45:10.387779+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388226+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:45:10.388226+0000","last_peered":"2024-09-13T14:45:10.388226+0000","last_clean":"2024-09-13T14:45:10.388226+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:45:10.388226+0000","last_undegraded":"2024-09-13T14:45:10.388226+0000","last_fullsized":"2024-09-13T14:45:10.388226+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"352'7548","reported_seq":7659,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:24.979739+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:45:24.979739+0000","last_peered":"2024-09-13T14:45:24.979739+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:45:24.979739+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7548,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1856}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":160,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345909+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:45:10.345909+0000","last_peered":"2024-09-13T14:45:10.345909+0000","last_clean":"2024-09-13T14:45:10.345909+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:45:10.345909+0000","last_undegraded":"2024-09-13T14:45:10.345909+0000","last_fullsized":"2024-09-13T14:45:10.345909+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":589,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.412882+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:45:20.412882+0000","last_peered":"2024-09-13T14:45:20.412882+0000","last_clean":"2024-09-13T14:45:20.412882+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:45:20.412882+0000","last_undegraded":"2024-09-13T14:45:20.412882+0000","last_fullsized":"2024-09-13T14:45:20.412882+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":551,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415106+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:45:20.415106+0000","last_peered":"2024-09-13T14:45:20.415106+0000","last_clean":"2024-09-13T14:45:20.415106+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:45:20.415106+0000","last_undegraded":"2024-09-13T14:45:20.415106+0000","last_fullsized":"2024-09-13T14:45:20.415106+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"352'7587","reported_seq":7651,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:27.176624+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:45:27.176624+0000","last_peered":"2024-09-13T14:45:27.176624+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:45:27.176624+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238092288,"num_objects":3633,"num_object_clones":0,"num_object_copies":10899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7587,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1863}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":454,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522636+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:45:10.522636+0000","last_peered":"2024-09-13T14:45:10.522636+0000","last_clean":"2024-09-13T14:45:10.522636+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:45:10.522636+0000","last_undegraded":"2024-09-13T14:45:10.522636+0000","last_fullsized":"2024-09-13T14:45:10.522636+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":876,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345205+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:45:10.345205+0000","last_peered":"2024-09-13T14:45:10.345205+0000","last_clean":"2024-09-13T14:45:10.345205+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:45:10.345205+0000","last_undegraded":"2024-09-13T14:45:10.345205+0000","last_fullsized":"2024-09-13T14:45:10.345205+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":543,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404544+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:45:20.404544+0000","last_peered":"2024-09-13T14:45:20.404544+0000","last_clean":"2024-09-13T14:45:20.404544+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:45:20.404544+0000","last_undegraded":"2024-09-13T14:45:20.404544+0000","last_fullsized":"2024-09-13T14:45:20.404544+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345062+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:45:10.345062+0000","last_peered":"2024-09-13T14:45:10.345062+0000","last_clean":"2024-09-13T14:45:10.345062+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:45:10.345062+0000","last_undegraded":"2024-09-13T14:45:10.345062+0000","last_fullsized":"2024-09-13T14:45:10.345062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"352'7840","reported_seq":7949,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:24.783064+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:45:24.783064+0000","last_peered":"2024-09-13T14:45:24.783064+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:45:24.783064+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243007488,"num_objects":3708,"num_object_clones":0,"num_object_copies":11124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7840,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-1972}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":397,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.416604+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:45:20.416604+0000","last_peered":"2024-09-13T14:45:20.416604+0000","last_clean":"2024-09-13T14:45:20.416604+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:45:20.416604+0000","last_undegraded":"2024-09-13T14:45:20.416604+0000","last_fullsized":"2024-09-13T14:45:20.416604+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":742,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522600+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:45:10.522600+0000","last_peered":"2024-09-13T14:45:10.522600+0000","last_clean":"2024-09-13T14:45:10.522600+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:45:10.522600+0000","last_undegraded":"2024-09-13T14:45:10.522600+0000","last_fullsized":"2024-09-13T14:45:10.522600+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":592,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390486+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:45:20.390486+0000","last_peered":"2024-09-13T14:45:20.390486+0000","last_clean":"2024-09-13T14:45:20.390486+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:45:20.390486+0000","last_undegraded":"2024-09-13T14:45:20.390486+0000","last_fullsized":"2024-09-13T14:45:20.390486+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":597,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390632+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:45:20.390632+0000","last_peered":"2024-09-13T14:45:20.390632+0000","last_clean":"2024-09-13T14:45:20.390632+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:45:20.390632+0000","last_undegraded":"2024-09-13T14:45:20.390632+0000","last_fullsized":"2024-09-13T14:45:20.390632+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"352'7829","reported_seq":17481,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:25.032329+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:45:25.032329+0000","last_peered":"2024-09-13T14:45:25.032329+0000","last_clean":"2024-09-13T14:45:25.032329+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:45:25.032329+0000","last_undegraded":"2024-09-13T14:45:25.032329+0000","last_fullsized":"2024-09-13T14:45:25.032329+0000","mapping_epoch":330,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7829,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":483,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522613+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:45:10.522613+0000","last_peered":"2024-09-13T14:45:10.522613+0000","last_clean":"2024-09-13T14:45:10.522613+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:45:10.522613+0000","last_undegraded":"2024-09-13T14:45:10.522613+0000","last_fullsized":"2024-09-13T14:45:10.522613+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":560,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404657+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:45:20.404657+0000","last_peered":"2024-09-13T14:45:20.404657+0000","last_clean":"2024-09-13T14:45:20.404657+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:45:20.404657+0000","last_undegraded":"2024-09-13T14:45:20.404657+0000","last_fullsized":"2024-09-13T14:45:20.404657+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":529,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.414353+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:45:20.414353+0000","last_peered":"2024-09-13T14:45:20.414353+0000","last_clean":"2024-09-13T14:45:20.414353+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:45:20.414353+0000","last_undegraded":"2024-09-13T14:45:20.414353+0000","last_fullsized":"2024-09-13T14:45:20.414353+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344681+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:45:10.344681+0000","last_peered":"2024-09-13T14:45:10.344681+0000","last_clean":"2024-09-13T14:45:10.344681+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:45:10.344681+0000","last_undegraded":"2024-09-13T14:45:10.344681+0000","last_fullsized":"2024-09-13T14:45:10.344681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"352'7551","reported_seq":7661,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:25.536300+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:45:25.536300+0000","last_peered":"2024-09-13T14:45:25.536300+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:45:25.536300+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241106944,"num_objects":3679,"num_object_clones":0,"num_object_copies":11037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7551,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1830}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":160,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345015+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:45:10.345015+0000","last_peered":"2024-09-13T14:45:10.345015+0000","last_clean":"2024-09-13T14:45:10.345015+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:45:10.345015+0000","last_undegraded":"2024-09-13T14:45:10.345015+0000","last_fullsized":"2024-09-13T14:45:10.345015+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":967,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345947+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:45:10.345947+0000","last_peered":"2024-09-13T14:45:10.345947+0000","last_clean":"2024-09-13T14:45:10.345947+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:45:10.345947+0000","last_undegraded":"2024-09-13T14:45:10.345947+0000","last_fullsized":"2024-09-13T14:45:10.345947+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388130+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:45:10.388130+0000","last_peered":"2024-09-13T14:45:10.388130+0000","last_clean":"2024-09-13T14:45:10.388130+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:45:10.388130+0000","last_undegraded":"2024-09-13T14:45:10.388130+0000","last_fullsized":"2024-09-13T14:45:10.388130+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":550,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.414423+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:45:20.414423+0000","last_peered":"2024-09-13T14:45:20.414423+0000","last_clean":"2024-09-13T14:45:20.414423+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:45:20.414423+0000","last_undegraded":"2024-09-13T14:45:20.414423+0000","last_fullsized":"2024-09-13T14:45:20.414423+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"352'7719","reported_seq":17110,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:28.558723+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:45:28.558723+0000","last_peered":"2024-09-13T14:45:28.558723+0000","last_clean":"2024-09-13T14:45:28.558723+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:45:28.558723+0000","last_undegraded":"2024-09-13T14:45:28.558723+0000","last_fullsized":"2024-09-13T14:45:28.558723+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238616576,"num_objects":3641,"num_object_clones":0,"num_object_copies":10923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344392+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:45:10.344392+0000","last_peered":"2024-09-13T14:45:10.344392+0000","last_clean":"2024-09-13T14:45:10.344392+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:45:10.344392+0000","last_undegraded":"2024-09-13T14:45:10.344392+0000","last_fullsized":"2024-09-13T14:45:10.344392+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344375+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:45:10.344375+0000","last_peered":"2024-09-13T14:45:10.344375+0000","last_clean":"2024-09-13T14:45:10.344375+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:45:10.344375+0000","last_undegraded":"2024-09-13T14:45:10.344375+0000","last_fullsized":"2024-09-13T14:45:10.344375+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1129,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415446+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:45:20.415446+0000","last_peered":"2024-09-13T14:45:20.415446+0000","last_clean":"2024-09-13T14:45:20.415446+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:45:20.415446+0000","last_undegraded":"2024-09-13T14:45:20.415446+0000","last_fullsized":"2024-09-13T14:45:20.415446+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":531,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.416102+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:45:20.416102+0000","last_peered":"2024-09-13T14:45:20.416102+0000","last_clean":"2024-09-13T14:45:20.416102+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:45:20.416102+0000","last_undegraded":"2024-09-13T14:45:20.416102+0000","last_fullsized":"2024-09-13T14:45:20.416102+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"352'7566","reported_seq":7677,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:24.999810+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:45:24.999810+0000","last_peered":"2024-09-13T14:45:24.999810+0000","last_clean":"2024-09-13T14:45:24.999810+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:45:24.999810+0000","last_undegraded":"2024-09-13T14:45:24.999810+0000","last_fullsized":"2024-09-13T14:45:24.999810+0000","mapping_epoch":303,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7066,"log_dups_size":0,"ondisk_log_size":7066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238419968,"num_objects":3638,"num_object_clones":0,"num_object_copies":10914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7566,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344376+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:45:10.344376+0000","last_peered":"2024-09-13T14:45:10.344376+0000","last_clean":"2024-09-13T14:45:10.344376+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:45:10.344376+0000","last_undegraded":"2024-09-13T14:45:10.344376+0000","last_fullsized":"2024-09-13T14:45:10.344376+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":540,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415373+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:45:20.415373+0000","last_peered":"2024-09-13T14:45:20.415373+0000","last_clean":"2024-09-13T14:45:20.415373+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:45:20.415373+0000","last_undegraded":"2024-09-13T14:45:20.415373+0000","last_fullsized":"2024-09-13T14:45:20.415373+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":746,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522998+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:45:10.522998+0000","last_peered":"2024-09-13T14:45:10.522998+0000","last_clean":"2024-09-13T14:45:10.522998+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:45:10.522998+0000","last_undegraded":"2024-09-13T14:45:10.522998+0000","last_fullsized":"2024-09-13T14:45:10.522998+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390152+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:45:20.390152+0000","last_peered":"2024-09-13T14:45:20.390152+0000","last_clean":"2024-09-13T14:45:20.390152+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:45:20.390152+0000","last_undegraded":"2024-09-13T14:45:20.390152+0000","last_fullsized":"2024-09-13T14:45:20.390152+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"352'7950","reported_seq":8061,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:28.643414+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:45:28.643414+0000","last_peered":"2024-09-13T14:45:28.643414+0000","last_clean":"2024-09-13T14:45:28.643414+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:45:28.643414+0000","last_undegraded":"2024-09-13T14:45:28.643414+0000","last_fullsized":"2024-09-13T14:45:28.643414+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7950,"log_dups_size":0,"ondisk_log_size":7950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7950,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388242+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:45:10.388242+0000","last_peered":"2024-09-13T14:45:10.388242+0000","last_clean":"2024-09-13T14:45:10.388242+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:45:10.388242+0000","last_undegraded":"2024-09-13T14:45:10.388242+0000","last_fullsized":"2024-09-13T14:45:10.388242+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.405638+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:45:20.405638+0000","last_peered":"2024-09-13T14:45:20.405638+0000","last_clean":"2024-09-13T14:45:20.405638+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:45:20.405638+0000","last_undegraded":"2024-09-13T14:45:20.405638+0000","last_fullsized":"2024-09-13T14:45:20.405638+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":915,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415204+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:45:20.415204+0000","last_peered":"2024-09-13T14:45:20.415204+0000","last_clean":"2024-09-13T14:45:20.415204+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:45:20.415204+0000","last_undegraded":"2024-09-13T14:45:20.415204+0000","last_fullsized":"2024-09-13T14:45:20.415204+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":573,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.412674+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:45:20.412674+0000","last_peered":"2024-09-13T14:45:20.412674+0000","last_clean":"2024-09-13T14:45:20.412674+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:45:20.412674+0000","last_undegraded":"2024-09-13T14:45:20.412674+0000","last_fullsized":"2024-09-13T14:45:20.412674+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"352'7878","reported_seq":19241,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:27.490995+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:45:27.490995+0000","last_peered":"2024-09-13T14:45:27.490995+0000","last_clean":"2024-09-13T14:45:27.490995+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:45:27.490995+0000","last_undegraded":"2024-09-13T14:45:27.490995+0000","last_fullsized":"2024-09-13T14:45:27.490995+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7878,"log_dups_size":0,"ondisk_log_size":7878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238223384,"num_objects":3636,"num_object_clones":0,"num_object_copies":10908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3636,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7878,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":427,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.405241+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:45:20.405241+0000","last_peered":"2024-09-13T14:45:20.405241+0000","last_clean":"2024-09-13T14:45:20.405241+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:45:20.405241+0000","last_undegraded":"2024-09-13T14:45:20.405241+0000","last_fullsized":"2024-09-13T14:45:20.405241+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":611,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390392+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:45:20.390392+0000","last_peered":"2024-09-13T14:45:20.390392+0000","last_clean":"2024-09-13T14:45:20.390392+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:45:20.390392+0000","last_undegraded":"2024-09-13T14:45:20.390392+0000","last_fullsized":"2024-09-13T14:45:20.390392+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":770,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344536+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:45:10.344536+0000","last_peered":"2024-09-13T14:45:10.344536+0000","last_clean":"2024-09-13T14:45:10.344536+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:45:10.344536+0000","last_undegraded":"2024-09-13T14:45:10.344536+0000","last_fullsized":"2024-09-13T14:45:10.344536+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387895+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:45:10.387895+0000","last_peered":"2024-09-13T14:45:10.387895+0000","last_clean":"2024-09-13T14:45:10.387895+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:45:10.387895+0000","last_undegraded":"2024-09-13T14:45:10.387895+0000","last_fullsized":"2024-09-13T14:45:10.387895+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388174+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:45:10.388174+0000","last_peered":"2024-09-13T14:45:10.388174+0000","last_clean":"2024-09-13T14:45:10.388174+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:45:10.388174+0000","last_undegraded":"2024-09-13T14:45:10.388174+0000","last_fullsized":"2024-09-13T14:45:10.388174+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":61,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388006+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:45:10.388006+0000","last_peered":"2024-09-13T14:45:10.388006+0000","last_clean":"2024-09-13T14:45:10.388006+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:45:10.388006+0000","last_undegraded":"2024-09-13T14:45:10.388006+0000","last_fullsized":"2024-09-13T14:45:10.388006+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":527,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415897+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:45:20.415897+0000","last_peered":"2024-09-13T14:45:20.415897+0000","last_clean":"2024-09-13T14:45:20.415897+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:45:20.415897+0000","last_undegraded":"2024-09-13T14:45:20.415897+0000","last_fullsized":"2024-09-13T14:45:20.415897+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522896+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:45:10.522896+0000","last_peered":"2024-09-13T14:45:10.522896+0000","last_clean":"2024-09-13T14:45:10.522896+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:45:10.522896+0000","last_undegraded":"2024-09-13T14:45:10.522896+0000","last_fullsized":"2024-09-13T14:45:10.522896+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344612+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:45:10.344612+0000","last_peered":"2024-09-13T14:45:10.344612+0000","last_clean":"2024-09-13T14:45:10.344612+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:45:10.344612+0000","last_undegraded":"2024-09-13T14:45:10.344612+0000","last_fullsized":"2024-09-13T14:45:10.344612+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":532,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404409+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:45:20.404409+0000","last_peered":"2024-09-13T14:45:20.404409+0000","last_clean":"2024-09-13T14:45:20.404409+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:45:20.404409+0000","last_undegraded":"2024-09-13T14:45:20.404409+0000","last_fullsized":"2024-09-13T14:45:20.404409+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":583,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390178+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:45:20.390178+0000","last_peered":"2024-09-13T14:45:20.390178+0000","last_clean":"2024-09-13T14:45:20.390178+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:45:20.390178+0000","last_undegraded":"2024-09-13T14:45:20.390178+0000","last_fullsized":"2024-09-13T14:45:20.390178+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":588,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.412570+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:45:20.412570+0000","last_peered":"2024-09-13T14:45:20.412570+0000","last_clean":"2024-09-13T14:45:20.412570+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:45:20.412570+0000","last_undegraded":"2024-09-13T14:45:20.412570+0000","last_fullsized":"2024-09-13T14:45:20.412570+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":349,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344459+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:45:10.344459+0000","last_peered":"2024-09-13T14:45:10.344459+0000","last_clean":"2024-09-13T14:45:10.344459+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:45:10.344459+0000","last_undegraded":"2024-09-13T14:45:10.344459+0000","last_fullsized":"2024-09-13T14:45:10.344459+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":605,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.412769+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:45:20.412769+0000","last_peered":"2024-09-13T14:45:20.412769+0000","last_clean":"2024-09-13T14:45:20.412769+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:45:20.412769+0000","last_undegraded":"2024-09-13T14:45:20.412769+0000","last_fullsized":"2024-09-13T14:45:20.412769+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390500+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:45:20.390500+0000","last_peered":"2024-09-13T14:45:20.390500+0000","last_clean":"2024-09-13T14:45:20.390500+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:45:20.390500+0000","last_undegraded":"2024-09-13T14:45:20.390500+0000","last_fullsized":"2024-09-13T14:45:20.390500+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":552,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.414539+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:45:20.414539+0000","last_peered":"2024-09-13T14:45:20.414539+0000","last_clean":"2024-09-13T14:45:20.414539+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:45:20.414539+0000","last_undegraded":"2024-09-13T14:45:20.414539+0000","last_fullsized":"2024-09-13T14:45:20.414539+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":601,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404570+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:45:20.404570+0000","last_peered":"2024-09-13T14:45:20.404570+0000","last_clean":"2024-09-13T14:45:20.404570+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:45:20.404570+0000","last_undegraded":"2024-09-13T14:45:20.404570+0000","last_fullsized":"2024-09-13T14:45:20.404570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.394468+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:45:10.394468+0000","last_peered":"2024-09-13T14:45:10.394468+0000","last_clean":"2024-09-13T14:45:10.394468+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:45:10.394468+0000","last_undegraded":"2024-09-13T14:45:10.394468+0000","last_fullsized":"2024-09-13T14:45:10.394468+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":498,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.415558+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:45:20.415558+0000","last_peered":"2024-09-13T14:45:20.415558+0000","last_clean":"2024-09-13T14:45:20.415558+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:45:20.415558+0000","last_undegraded":"2024-09-13T14:45:20.415558+0000","last_fullsized":"2024-09-13T14:45:20.415558+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":579,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390022+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:45:20.390022+0000","last_peered":"2024-09-13T14:45:20.390022+0000","last_clean":"2024-09-13T14:45:20.390022+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:45:20.390022+0000","last_undegraded":"2024-09-13T14:45:20.390022+0000","last_fullsized":"2024-09-13T14:45:20.390022+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388058+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:45:10.388058+0000","last_peered":"2024-09-13T14:45:10.388058+0000","last_clean":"2024-09-13T14:45:10.388058+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:45:10.388058+0000","last_undegraded":"2024-09-13T14:45:10.388058+0000","last_fullsized":"2024-09-13T14:45:10.388058+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344870+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:45:10.344870+0000","last_peered":"2024-09-13T14:45:10.344870+0000","last_clean":"2024-09-13T14:45:10.344870+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:45:10.344870+0000","last_undegraded":"2024-09-13T14:45:10.344870+0000","last_fullsized":"2024-09-13T14:45:10.344870+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.394419+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:45:10.394419+0000","last_peered":"2024-09-13T14:45:10.394419+0000","last_clean":"2024-09-13T14:45:10.394419+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:45:10.394419+0000","last_undegraded":"2024-09-13T14:45:10.394419+0000","last_fullsized":"2024-09-13T14:45:10.394419+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":283,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344535+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:45:10.344535+0000","last_peered":"2024-09-13T14:45:10.344535+0000","last_clean":"2024-09-13T14:45:10.344535+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:45:10.344535+0000","last_undegraded":"2024-09-13T14:45:10.344535+0000","last_fullsized":"2024-09-13T14:45:10.344535+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":597,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404751+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:45:20.404751+0000","last_peered":"2024-09-13T14:45:20.404751+0000","last_clean":"2024-09-13T14:45:20.404751+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:45:20.404751+0000","last_undegraded":"2024-09-13T14:45:20.404751+0000","last_fullsized":"2024-09-13T14:45:20.404751+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":584,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.404867+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:45:20.404867+0000","last_peered":"2024-09-13T14:45:20.404867+0000","last_clean":"2024-09-13T14:45:20.404867+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:45:20.404867+0000","last_undegraded":"2024-09-13T14:45:20.404867+0000","last_fullsized":"2024-09-13T14:45:20.404867+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387750+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:45:10.387750+0000","last_peered":"2024-09-13T14:45:10.387750+0000","last_clean":"2024-09-13T14:45:10.387750+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:45:10.387750+0000","last_undegraded":"2024-09-13T14:45:10.387750+0000","last_fullsized":"2024-09-13T14:45:10.387750+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":589,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390010+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:45:20.390010+0000","last_peered":"2024-09-13T14:45:20.390010+0000","last_clean":"2024-09-13T14:45:20.390010+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:45:20.390010+0000","last_undegraded":"2024-09-13T14:45:20.390010+0000","last_fullsized":"2024-09-13T14:45:20.390010+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3867607064,"num_objects":59016,"num_object_clones":0,"num_object_copies":177048,"num_objects_missing_on_primary":2679,"num_objects_missing":2679,"num_objects_degraded":36985,"num_objects_misplaced":3631,"num_objects_unfound":0,"num_objects_dirty":59016,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123662,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23798,"num_bytes_recovered":1499201536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10662916096,"data_stored":10662903880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120062,"ondisk_log_size":120062,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397912,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2093180,"kb_used_data":1197172,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":91651972,"statfs":{"total":95995035648,"available":93851619328,"internally_reserved":0,"allocated":1225904128,"data_stored":1224049025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594140,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2132712,"kb_used_data":1538656,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91612440,"statfs":{"total":95995035648,"available":93811138560,"internally_reserved":0,"allocated":1575583744,"data_stored":1573748395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665594,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617192,"kb_used_data":284640,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93127960,"statfs":{"total":95995035648,"available":95363031040,"internally_reserved":0,"allocated":291471360,"data_stored":289632342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986595,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1754420,"kb_used_data":858732,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":91990732,"statfs":{"total":95995035648,"available":94198509568,"internally_reserved":0,"allocated":879341568,"data_stored":877494563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,3,0,1,2,6,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697155,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527972,"kb_used_data":1796188,"kb_used_omap":4,"kb_used_meta":731771,"kb_avail":91217180,"statfs":{"total":95995035648,"available":93406392320,"internally_reserved":0,"allocated":1839296512,"data_stored":1837461647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":749333579},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860682,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2327276,"kb_used_data":1710180,"kb_used_omap":8,"kb_used_meta":617079,"kb_avail":91417876,"statfs":{"total":95995035648,"available":93611905024,"internally_reserved":0,"allocated":1751224320,"data_stored":1749381801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":631889740},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020353,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1968224,"kb_used_data":1664664,"kb_used_omap":9,"kb_used_meta":303542,"kb_avail":91776928,"statfs":{"total":95995035648,"available":93979574272,"internally_reserved":0,"allocated":1704615936,"data_stored":1702784544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":310827845},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187744,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2039968,"kb_used_data":1421144,"kb_used_omap":7,"kb_used_meta":618808,"kb_avail":91705184,"statfs":{"total":95995035648,"available":93906108416,"internally_reserved":0,"allocated":1455251456,"data_stored":1453420330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":633659723},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1445003264,"data_stored":1445003264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1695154176,"data_stored":1695154176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1744441344,"data_stored":1744437272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1833172992,"data_stored":1833172992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":873725952,"data_stored":873725952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":286003200,"data_stored":285999128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1570766848,"data_stored":1570766848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1214648320,"data_stored":1214644248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:31.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:31.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:31.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:31.661 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:31.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:31.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:31.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:31.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:31.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:31.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:31.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:32.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:32.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:32.138 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:32.138 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:32.178 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:32.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:32.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:32 smithi017 ceph-mon[103001]: pgmap v1093: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 36942/176163 objects degraded (20.970%); 3588/176163 objects misplaced (2.037%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:45:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:32 smithi017 ceph-mon[107556]: pgmap v1093: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 36942/176163 objects degraded (20.970%); 3588/176163 objects misplaced (2.037%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:45:32.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:32 smithi189 ceph-mon[90992]: pgmap v1093: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 36942/176163 objects degraded (20.970%); 3588/176163 objects misplaced (2.037%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:45:32.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:32.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:32.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:32.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:32.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:32.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:32.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:32.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:33.038 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:45:33.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:33.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:33.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:33.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:33.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:33.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:33.371 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:33.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.495 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:33.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:33.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.703 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:33.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:33.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:33.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:33.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:33.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:33.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:34.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:34.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:34.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:34.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:34.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:34.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:34.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:34.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:34.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:34.526 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:45:34.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:34.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:34.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:34 smithi189 ceph-mon[90992]: pgmap v1094: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36854/175674 objects degraded (20.979%); 3588/175674 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:34.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36854/175674 objects degraded (20.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:34.639 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:34.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:34.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:34.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:34.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:34.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[103001]: pgmap v1094: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36854/175674 objects degraded (20.979%); 3588/175674 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36854/175674 objects degraded (20.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[107556]: pgmap v1094: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36854/175674 objects degraded (20.979%); 3588/175674 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36854/175674 objects degraded (20.979%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:34.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:34.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:35.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:35.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:35.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:35.225 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:35.225 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:35.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:35.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.491 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:35.616 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[103001]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:45:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:35 smithi017 ceph-mon[107556]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:45:35.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:35.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:35.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:35.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:35.968 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:35.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:35 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:35.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/277043665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:35.973 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:35 smithi189 ceph-mon[90992]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:45:35.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:35.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:36.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:45:36.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:36.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:36.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.343 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:36.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:36.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:36.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:36 smithi017 ceph-mon[103001]: pgmap v1096: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 36812/174588 objects degraded (21.085%); 3547/174588 objects misplaced (2.032%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:36.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:36 smithi017 ceph-mon[107556]: pgmap v1096: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 36812/174588 objects degraded (21.085%); 3547/174588 objects misplaced (2.032%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:36.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:36 smithi189 ceph-mon[90992]: pgmap v1096: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 36812/174588 objects degraded (21.085%); 3547/174588 objects misplaced (2.032%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:45:36.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:37.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:37.101 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:45:37.101 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:37.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:37.122 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:45:37.122 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:37.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:37.138 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:45:37.139 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:37.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:45:37.160 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:37.175 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:45:37.176 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:37.192 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:45:37.192 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:37.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:45:37.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:37.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:45:37.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:37.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:37.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:37.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:37.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:37.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:37.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:37] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-13T14:45:37.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:37.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:37.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:37.759 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:37.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2531213412' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:37.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:37.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:37.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:45:37.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2531213412' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:37.897 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2531213412' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:37.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:38.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:38.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:38.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:38.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:38.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:38.264 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:38.269 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:45:38.270 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:45:38.270 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:45:38.270 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:45:38.270 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:45:38.367 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:38.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:38.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:38.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:38 smithi017 ceph-mon[103001]: pgmap v1097: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 36736/174006 objects degraded (21.112%); 3547/174006 objects misplaced (2.038%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:38.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:38 smithi017 ceph-mon[107556]: pgmap v1097: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 36736/174006 objects degraded (21.112%); 3547/174006 objects misplaced (2.038%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:38.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:38.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:38.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:38.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:38.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:38.925 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:38 smithi189 ceph-mon[90992]: pgmap v1097: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 36736/174006 objects degraded (21.112%); 3547/174006 objects misplaced (2.038%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:38.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:45:38.958 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:38.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:39.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:39.116 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:39.117 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:45:39.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:39.143 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:39.238 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:39.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:39.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:39.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:39.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:39.497 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:39.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:39.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:39.641 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:39.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:39.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36736/174006 objects degraded (21.112%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:39.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:39.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36736/174006 objects degraded (21.112%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:39.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36736/174006 objects degraded (21.112%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:39.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:39.877 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1097,"stamp":"2024-09-13T14:45:37.226521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3804007094,"num_objects":58002,"num_object_clones":0,"num_object_copies":174006,"num_objects_missing_on_primary":2594,"num_objects_missing":2594,"num_objects_degraded":36736,"num_objects_misplaced":3547,"num_objects_unfound":0,"num_objects_dirty":58002,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":125651,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24021,"num_bytes_recovered":1510246557,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122436,"ondisk_log_size":122436,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15334984,"kb_used_data":10314880,"kb_used_omap":44,"kb_used_meta":5019987,"kb_avail":734626232,"statfs":{"total":767960285184,"available":752257261568,"internally_reserved":0,"allocated":10562437120,"data_stored":10547697202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5140466935},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85131264,"num_objects":-1299,"num_object_clones":0,"num_object_copies":-3897,"num_objects_missing_on_primary":-85,"num_objects_missing":-85,"num_objects_degraded":-249,"num_objects_misplaced":-84,"num_objects_unfound":0,"num_objects_dirty":-1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1299,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002857"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":590,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.534323+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:45:34.534323+0000","last_peered":"2024-09-13T14:45:34.534323+0000","last_clean":"2024-09-13T14:45:34.534323+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:45:34.534323+0000","last_undegraded":"2024-09-13T14:45:34.534323+0000","last_fullsized":"2024-09-13T14:45:34.534323+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":350,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522743+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:45:10.522743+0000","last_peered":"2024-09-13T14:45:10.522743+0000","last_clean":"2024-09-13T14:45:10.522743+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:45:10.522743+0000","last_undegraded":"2024-09-13T14:45:10.522743+0000","last_fullsized":"2024-09-13T14:45:10.522743+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344469+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:45:10.344469+0000","last_peered":"2024-09-13T14:45:10.344469+0000","last_clean":"2024-09-13T14:45:10.344469+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:45:10.344469+0000","last_undegraded":"2024-09-13T14:45:10.344469+0000","last_fullsized":"2024-09-13T14:45:10.344469+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":573,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.392395+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:45:20.392395+0000","last_peered":"2024-09-13T14:45:20.392395+0000","last_clean":"2024-09-13T14:45:20.392395+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:45:20.392395+0000","last_undegraded":"2024-09-13T14:45:20.392395+0000","last_fullsized":"2024-09-13T14:45:20.392395+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":573,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390497+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:45:20.390497+0000","last_peered":"2024-09-13T14:45:20.390497+0000","last_clean":"2024-09-13T14:45:20.390497+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:45:20.390497+0000","last_undegraded":"2024-09-13T14:45:20.390497+0000","last_fullsized":"2024-09-13T14:45:20.390497+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":319,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522740+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:45:10.522740+0000","last_peered":"2024-09-13T14:45:10.522740+0000","last_clean":"2024-09-13T14:45:10.522740+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:45:10.522740+0000","last_undegraded":"2024-09-13T14:45:10.522740+0000","last_fullsized":"2024-09-13T14:45:10.522740+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":589,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.533799+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:45:34.533799+0000","last_peered":"2024-09-13T14:45:34.533799+0000","last_clean":"2024-09-13T14:45:34.533799+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:45:34.533799+0000","last_undegraded":"2024-09-13T14:45:34.533799+0000","last_fullsized":"2024-09-13T14:45:34.533799+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":341,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.343961+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:45:10.343961+0000","last_peered":"2024-09-13T14:45:10.343961+0000","last_clean":"2024-09-13T14:45:10.343961+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:45:10.343961+0000","last_undegraded":"2024-09-13T14:45:10.343961+0000","last_fullsized":"2024-09-13T14:45:10.343961+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"353'7756","reported_seq":14196,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:37.042504+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:45:37.042504+0000","last_peered":"2024-09-13T14:45:37.042504+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:45:37.042504+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7756,"log_dups_size":0,"ondisk_log_size":7756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5983,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7756,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2111,"num_bytes_recovered":138346496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2550}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":345,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522989+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:45:10.522989+0000","last_peered":"2024-09-13T14:45:10.522989+0000","last_clean":"2024-09-13T14:45:10.522989+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:45:10.522989+0000","last_undegraded":"2024-09-13T14:45:10.522989+0000","last_fullsized":"2024-09-13T14:45:10.522989+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":264,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344045+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:45:10.344045+0000","last_peered":"2024-09-13T14:45:10.344045+0000","last_clean":"2024-09-13T14:45:10.344045+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:45:10.344045+0000","last_undegraded":"2024-09-13T14:45:10.344045+0000","last_fullsized":"2024-09-13T14:45:10.344045+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"352'7797","reported_seq":17346,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:32.667582+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:45:32.667582+0000","last_peered":"2024-09-13T14:45:32.667582+0000","last_clean":"2024-09-13T14:45:32.667582+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:45:32.667582+0000","last_undegraded":"2024-09-13T14:45:32.667582+0000","last_fullsized":"2024-09-13T14:45:32.667582+0000","mapping_epoch":326,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7597,"log_dups_size":0,"ondisk_log_size":7597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241893376,"num_objects":3691,"num_object_clones":0,"num_object_copies":11073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7797,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":363,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.523031+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:45:10.523031+0000","last_peered":"2024-09-13T14:45:10.523031+0000","last_clean":"2024-09-13T14:45:10.523031+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:45:10.523031+0000","last_undegraded":"2024-09-13T14:45:10.523031+0000","last_fullsized":"2024-09-13T14:45:10.523031+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387964+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:45:10.387964+0000","last_peered":"2024-09-13T14:45:10.387964+0000","last_clean":"2024-09-13T14:45:10.387964+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:45:10.387964+0000","last_undegraded":"2024-09-13T14:45:10.387964+0000","last_fullsized":"2024-09-13T14:45:10.387964+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"352'8119","reported_seq":8230,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:33.692951+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:45:33.692951+0000","last_peered":"2024-09-13T14:45:33.692951+0000","last_clean":"2024-09-13T14:45:33.692951+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:45:33.692951+0000","last_undegraded":"2024-09-13T14:45:33.692951+0000","last_fullsized":"2024-09-13T14:45:33.692951+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8119,"log_dups_size":0,"ondisk_log_size":8119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241106944,"num_objects":3679,"num_object_clones":0,"num_object_copies":11037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8119,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.523002+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:45:10.523002+0000","last_peered":"2024-09-13T14:45:10.523002+0000","last_clean":"2024-09-13T14:45:10.523002+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:45:10.523002+0000","last_undegraded":"2024-09-13T14:45:10.523002+0000","last_fullsized":"2024-09-13T14:45:10.523002+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":332,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344009+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:45:10.344009+0000","last_peered":"2024-09-13T14:45:10.344009+0000","last_clean":"2024-09-13T14:45:10.344009+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:45:10.344009+0000","last_undegraded":"2024-09-13T14:45:10.344009+0000","last_fullsized":"2024-09-13T14:45:10.344009+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"353'7898","reported_seq":8011,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.589030+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:45:34.589030+0000","last_peered":"2024-09-13T14:45:34.589030+0000","last_clean":"2024-09-13T14:45:34.589030+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:45:34.589030+0000","last_undegraded":"2024-09-13T14:45:34.589030+0000","last_fullsized":"2024-09-13T14:45:34.589030+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7898,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":280,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344781+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:45:10.344781+0000","last_peered":"2024-09-13T14:45:10.344781+0000","last_clean":"2024-09-13T14:45:10.344781+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:45:10.344781+0000","last_undegraded":"2024-09-13T14:45:10.344781+0000","last_fullsized":"2024-09-13T14:45:10.344781+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":599,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390255+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:45:20.390255+0000","last_peered":"2024-09-13T14:45:20.390255+0000","last_clean":"2024-09-13T14:45:20.390255+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:45:20.390255+0000","last_undegraded":"2024-09-13T14:45:20.390255+0000","last_fullsized":"2024-09-13T14:45:20.390255+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"353'7824","reported_seq":7935,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:35.575998+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:45:35.575998+0000","last_peered":"2024-09-13T14:45:35.575998+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:45:35.575998+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238419968,"num_objects":3638,"num_object_clones":0,"num_object_copies":10914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7824,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-1995}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":463,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522542+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:45:10.522542+0000","last_peered":"2024-09-13T14:45:10.522542+0000","last_clean":"2024-09-13T14:45:10.522542+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:45:10.522542+0000","last_undegraded":"2024-09-13T14:45:10.522542+0000","last_fullsized":"2024-09-13T14:45:10.522542+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":599,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.533996+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:45:34.533996+0000","last_peered":"2024-09-13T14:45:34.533996+0000","last_clean":"2024-09-13T14:45:34.533996+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:45:34.533996+0000","last_undegraded":"2024-09-13T14:45:34.533996+0000","last_fullsized":"2024-09-13T14:45:34.533996+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388281+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:45:10.388281+0000","last_peered":"2024-09-13T14:45:10.388281+0000","last_clean":"2024-09-13T14:45:10.388281+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:45:10.388281+0000","last_undegraded":"2024-09-13T14:45:10.388281+0000","last_fullsized":"2024-09-13T14:45:10.388281+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"353'7541","reported_seq":7654,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.632215+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:45:34.632215+0000","last_peered":"2024-09-13T14:45:34.632215+0000","last_clean":"2024-09-13T14:45:34.632215+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:45:34.632215+0000","last_undegraded":"2024-09-13T14:45:34.632215+0000","last_fullsized":"2024-09-13T14:45:34.632215+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232849408,"num_objects":3553,"num_object_clones":0,"num_object_copies":10659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7541,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":167,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344121+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:45:10.344121+0000","last_peered":"2024-09-13T14:45:10.344121+0000","last_clean":"2024-09-13T14:45:10.344121+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:45:10.344121+0000","last_undegraded":"2024-09-13T14:45:10.344121+0000","last_fullsized":"2024-09-13T14:45:10.344121+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.343873+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:45:10.343873+0000","last_peered":"2024-09-13T14:45:10.343873+0000","last_clean":"2024-09-13T14:45:10.343873+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:45:10.343873+0000","last_undegraded":"2024-09-13T14:45:10.343873+0000","last_fullsized":"2024-09-13T14:45:10.343873+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":65,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388383+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:45:10.388383+0000","last_peered":"2024-09-13T14:45:10.388383+0000","last_clean":"2024-09-13T14:45:10.388383+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:45:10.388383+0000","last_undegraded":"2024-09-13T14:45:10.388383+0000","last_fullsized":"2024-09-13T14:45:10.388383+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"352'7761","reported_seq":13643,"reported_epoch":352,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:34.436493+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:45:34.436493+0000","last_peered":"2024-09-13T14:45:34.436493+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:45:34.436493+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239271936,"num_objects":3651,"num_object_clones":0,"num_object_copies":10953,"num_objects_missing_on_primary":2594,"num_objects_missing":2594,"num_objects_degraded":2969,"num_objects_misplaced":3547,"num_objects_unfound":0,"num_objects_dirty":3651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7761,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2056,"num_bytes_recovered":134742016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2594},{"shards":"0,3,4","objects":953},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387779+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:45:10.387779+0000","last_peered":"2024-09-13T14:45:10.387779+0000","last_clean":"2024-09-13T14:45:10.387779+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:45:10.387779+0000","last_undegraded":"2024-09-13T14:45:10.387779+0000","last_fullsized":"2024-09-13T14:45:10.387779+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388226+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:45:10.388226+0000","last_peered":"2024-09-13T14:45:10.388226+0000","last_clean":"2024-09-13T14:45:10.388226+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:45:10.388226+0000","last_undegraded":"2024-09-13T14:45:10.388226+0000","last_fullsized":"2024-09-13T14:45:10.388226+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"353'7621","reported_seq":7734,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:34.644175+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:45:34.644175+0000","last_peered":"2024-09-13T14:45:34.644175+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:45:34.644175+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231800832,"num_objects":3537,"num_object_clones":0,"num_object_copies":10611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7621,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1929}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":160,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345909+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:45:10.345909+0000","last_peered":"2024-09-13T14:45:10.345909+0000","last_clean":"2024-09-13T14:45:10.345909+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:45:10.345909+0000","last_undegraded":"2024-09-13T14:45:10.345909+0000","last_fullsized":"2024-09-13T14:45:10.345909+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":591,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542303+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:45:34.542303+0000","last_peered":"2024-09-13T14:45:34.542303+0000","last_clean":"2024-09-13T14:45:34.542303+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:45:34.542303+0000","last_undegraded":"2024-09-13T14:45:34.542303+0000","last_fullsized":"2024-09-13T14:45:34.542303+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":553,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.537923+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:45:34.537923+0000","last_peered":"2024-09-13T14:45:34.537923+0000","last_clean":"2024-09-13T14:45:34.537923+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:45:34.537923+0000","last_undegraded":"2024-09-13T14:45:34.537923+0000","last_fullsized":"2024-09-13T14:45:34.537923+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"352'7624","reported_seq":7688,"reported_epoch":352,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:32.653591+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:45:32.653591+0000","last_peered":"2024-09-13T14:45:32.653591+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:45:32.653591+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235667456,"num_objects":3596,"num_object_clones":0,"num_object_copies":10788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7624,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1900}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":454,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522636+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:45:10.522636+0000","last_peered":"2024-09-13T14:45:10.522636+0000","last_clean":"2024-09-13T14:45:10.522636+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:45:10.522636+0000","last_undegraded":"2024-09-13T14:45:10.522636+0000","last_fullsized":"2024-09-13T14:45:10.522636+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":876,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345205+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:45:10.345205+0000","last_peered":"2024-09-13T14:45:10.345205+0000","last_clean":"2024-09-13T14:45:10.345205+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:45:10.345205+0000","last_undegraded":"2024-09-13T14:45:10.345205+0000","last_fullsized":"2024-09-13T14:45:10.345205+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":545,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.533090+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:45:34.533090+0000","last_peered":"2024-09-13T14:45:34.533090+0000","last_clean":"2024-09-13T14:45:34.533090+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:45:34.533090+0000","last_undegraded":"2024-09-13T14:45:34.533090+0000","last_fullsized":"2024-09-13T14:45:34.533090+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345062+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:45:10.345062+0000","last_peered":"2024-09-13T14:45:10.345062+0000","last_clean":"2024-09-13T14:45:10.345062+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:45:10.345062+0000","last_undegraded":"2024-09-13T14:45:10.345062+0000","last_fullsized":"2024-09-13T14:45:10.345062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"353'7917","reported_seq":8028,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:34.564940+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:45:34.564940+0000","last_peered":"2024-09-13T14:45:34.564940+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:45:34.564940+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237961216,"num_objects":3631,"num_object_clones":0,"num_object_copies":10893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7917,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2049}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":399,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.539537+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:45:34.539537+0000","last_peered":"2024-09-13T14:45:34.539537+0000","last_clean":"2024-09-13T14:45:34.539537+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:45:34.539537+0000","last_undegraded":"2024-09-13T14:45:34.539537+0000","last_fullsized":"2024-09-13T14:45:34.539537+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":742,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522600+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:45:10.522600+0000","last_peered":"2024-09-13T14:45:10.522600+0000","last_clean":"2024-09-13T14:45:10.522600+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:45:10.522600+0000","last_undegraded":"2024-09-13T14:45:10.522600+0000","last_fullsized":"2024-09-13T14:45:10.522600+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":592,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390486+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:45:20.390486+0000","last_peered":"2024-09-13T14:45:20.390486+0000","last_clean":"2024-09-13T14:45:20.390486+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:45:20.390486+0000","last_undegraded":"2024-09-13T14:45:20.390486+0000","last_fullsized":"2024-09-13T14:45:20.390486+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":597,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390632+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:45:20.390632+0000","last_peered":"2024-09-13T14:45:20.390632+0000","last_clean":"2024-09-13T14:45:20.390632+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:45:20.390632+0000","last_undegraded":"2024-09-13T14:45:20.390632+0000","last_fullsized":"2024-09-13T14:45:20.390632+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"353'7905","reported_seq":17559,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:35.512358+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:45:35.512358+0000","last_peered":"2024-09-13T14:45:35.512358+0000","last_clean":"2024-09-13T14:45:35.512358+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:45:35.512358+0000","last_undegraded":"2024-09-13T14:45:35.512358+0000","last_fullsized":"2024-09-13T14:45:35.512358+0000","mapping_epoch":330,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7905,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":483,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522613+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:45:10.522613+0000","last_peered":"2024-09-13T14:45:10.522613+0000","last_clean":"2024-09-13T14:45:10.522613+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:45:10.522613+0000","last_undegraded":"2024-09-13T14:45:10.522613+0000","last_fullsized":"2024-09-13T14:45:10.522613+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":562,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.532828+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:45:34.532828+0000","last_peered":"2024-09-13T14:45:34.532828+0000","last_clean":"2024-09-13T14:45:34.532828+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:45:34.532828+0000","last_undegraded":"2024-09-13T14:45:34.532828+0000","last_fullsized":"2024-09-13T14:45:34.532828+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":531,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.536489+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:45:34.536489+0000","last_peered":"2024-09-13T14:45:34.536489+0000","last_clean":"2024-09-13T14:45:34.536489+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:45:34.536489+0000","last_undegraded":"2024-09-13T14:45:34.536489+0000","last_fullsized":"2024-09-13T14:45:34.536489+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344681+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:45:10.344681+0000","last_peered":"2024-09-13T14:45:10.344681+0000","last_clean":"2024-09-13T14:45:10.344681+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:45:10.344681+0000","last_undegraded":"2024-09-13T14:45:10.344681+0000","last_fullsized":"2024-09-13T14:45:10.344681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"353'7623","reported_seq":7735,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:35.540912+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:45:35.540912+0000","last_peered":"2024-09-13T14:45:35.540912+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:45:35.540912+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236388352,"num_objects":3607,"num_object_clones":0,"num_object_copies":10821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7623,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1902}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":160,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345015+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:45:10.345015+0000","last_peered":"2024-09-13T14:45:10.345015+0000","last_clean":"2024-09-13T14:45:10.345015+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:45:10.345015+0000","last_undegraded":"2024-09-13T14:45:10.345015+0000","last_fullsized":"2024-09-13T14:45:10.345015+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":967,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.345947+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:45:10.345947+0000","last_peered":"2024-09-13T14:45:10.345947+0000","last_clean":"2024-09-13T14:45:10.345947+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:45:10.345947+0000","last_undegraded":"2024-09-13T14:45:10.345947+0000","last_fullsized":"2024-09-13T14:45:10.345947+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388130+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:45:10.388130+0000","last_peered":"2024-09-13T14:45:10.388130+0000","last_clean":"2024-09-13T14:45:10.388130+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:45:10.388130+0000","last_undegraded":"2024-09-13T14:45:10.388130+0000","last_fullsized":"2024-09-13T14:45:10.388130+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":552,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.536562+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:45:34.536562+0000","last_peered":"2024-09-13T14:45:34.536562+0000","last_clean":"2024-09-13T14:45:34.536562+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:45:34.536562+0000","last_undegraded":"2024-09-13T14:45:34.536562+0000","last_fullsized":"2024-09-13T14:45:34.536562+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"352'7753","reported_seq":17144,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:33.636535+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:45:33.636535+0000","last_peered":"2024-09-13T14:45:33.636535+0000","last_clean":"2024-09-13T14:45:33.636535+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:45:33.636535+0000","last_undegraded":"2024-09-13T14:45:33.636535+0000","last_fullsized":"2024-09-13T14:45:33.636535+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7753,"log_dups_size":0,"ondisk_log_size":7753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236388352,"num_objects":3607,"num_object_clones":0,"num_object_copies":10821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7753,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344392+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:45:10.344392+0000","last_peered":"2024-09-13T14:45:10.344392+0000","last_clean":"2024-09-13T14:45:10.344392+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:45:10.344392+0000","last_undegraded":"2024-09-13T14:45:10.344392+0000","last_fullsized":"2024-09-13T14:45:10.344392+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344375+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:45:10.344375+0000","last_peered":"2024-09-13T14:45:10.344375+0000","last_clean":"2024-09-13T14:45:10.344375+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:45:10.344375+0000","last_undegraded":"2024-09-13T14:45:10.344375+0000","last_fullsized":"2024-09-13T14:45:10.344375+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1131,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.536726+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:45:34.536726+0000","last_peered":"2024-09-13T14:45:34.536726+0000","last_clean":"2024-09-13T14:45:34.536726+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:45:34.536726+0000","last_undegraded":"2024-09-13T14:45:34.536726+0000","last_fullsized":"2024-09-13T14:45:34.536726+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":533,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542289+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:45:34.542289+0000","last_peered":"2024-09-13T14:45:34.542289+0000","last_clean":"2024-09-13T14:45:34.542289+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:45:34.542289+0000","last_undegraded":"2024-09-13T14:45:34.542289+0000","last_fullsized":"2024-09-13T14:45:34.542289+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"353'7644","reported_seq":7757,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:35.557253+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:45:35.557253+0000","last_peered":"2024-09-13T14:45:35.557253+0000","last_clean":"2024-09-13T14:45:35.557253+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:45:35.557253+0000","last_undegraded":"2024-09-13T14:45:35.557253+0000","last_fullsized":"2024-09-13T14:45:35.557253+0000","mapping_epoch":303,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233308160,"num_objects":3560,"num_object_clones":0,"num_object_copies":10680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7644,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344376+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:45:10.344376+0000","last_peered":"2024-09-13T14:45:10.344376+0000","last_clean":"2024-09-13T14:45:10.344376+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:45:10.344376+0000","last_undegraded":"2024-09-13T14:45:10.344376+0000","last_fullsized":"2024-09-13T14:45:10.344376+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":542,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.536644+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:45:34.536644+0000","last_peered":"2024-09-13T14:45:34.536644+0000","last_clean":"2024-09-13T14:45:34.536644+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:45:34.536644+0000","last_undegraded":"2024-09-13T14:45:34.536644+0000","last_fullsized":"2024-09-13T14:45:34.536644+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":746,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522998+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:45:10.522998+0000","last_peered":"2024-09-13T14:45:10.522998+0000","last_clean":"2024-09-13T14:45:10.522998+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:45:10.522998+0000","last_undegraded":"2024-09-13T14:45:10.522998+0000","last_fullsized":"2024-09-13T14:45:10.522998+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390152+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:45:20.390152+0000","last_peered":"2024-09-13T14:45:20.390152+0000","last_clean":"2024-09-13T14:45:20.390152+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:45:20.390152+0000","last_undegraded":"2024-09-13T14:45:20.390152+0000","last_fullsized":"2024-09-13T14:45:20.390152+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"352'8000","reported_seq":8111,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:33.647684+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:45:33.647684+0000","last_peered":"2024-09-13T14:45:33.647684+0000","last_clean":"2024-09-13T14:45:33.647684+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:45:33.647684+0000","last_undegraded":"2024-09-13T14:45:33.647684+0000","last_fullsized":"2024-09-13T14:45:33.647684+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8000,"log_dups_size":0,"ondisk_log_size":8000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827840,"num_objects":3690,"num_object_clones":0,"num_object_copies":11070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8000,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388242+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:45:10.388242+0000","last_peered":"2024-09-13T14:45:10.388242+0000","last_clean":"2024-09-13T14:45:10.388242+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:45:10.388242+0000","last_undegraded":"2024-09-13T14:45:10.388242+0000","last_fullsized":"2024-09-13T14:45:10.388242+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":577,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.534083+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:45:34.534083+0000","last_peered":"2024-09-13T14:45:34.534083+0000","last_clean":"2024-09-13T14:45:34.534083+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:45:34.534083+0000","last_undegraded":"2024-09-13T14:45:34.534083+0000","last_fullsized":"2024-09-13T14:45:34.534083+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":918,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.537301+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:45:34.537301+0000","last_peered":"2024-09-13T14:45:34.537301+0000","last_clean":"2024-09-13T14:45:34.537301+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:45:34.537301+0000","last_undegraded":"2024-09-13T14:45:34.537301+0000","last_fullsized":"2024-09-13T14:45:34.537301+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":575,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542143+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:45:34.542143+0000","last_peered":"2024-09-13T14:45:34.542143+0000","last_clean":"2024-09-13T14:45:34.542143+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:45:34.542143+0000","last_undegraded":"2024-09-13T14:45:34.542143+0000","last_fullsized":"2024-09-13T14:45:34.542143+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"352'7921","reported_seq":19284,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:32.727722+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:45:32.727722+0000","last_peered":"2024-09-13T14:45:32.727722+0000","last_clean":"2024-09-13T14:45:32.727722+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:45:32.727722+0000","last_undegraded":"2024-09-13T14:45:32.727722+0000","last_fullsized":"2024-09-13T14:45:32.727722+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7921,"log_dups_size":0,"ondisk_log_size":7921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405336,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7921,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":429,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.534179+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:45:34.534179+0000","last_peered":"2024-09-13T14:45:34.534179+0000","last_clean":"2024-09-13T14:45:34.534179+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:45:34.534179+0000","last_undegraded":"2024-09-13T14:45:34.534179+0000","last_fullsized":"2024-09-13T14:45:34.534179+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":611,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390392+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:45:20.390392+0000","last_peered":"2024-09-13T14:45:20.390392+0000","last_clean":"2024-09-13T14:45:20.390392+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:45:20.390392+0000","last_undegraded":"2024-09-13T14:45:20.390392+0000","last_fullsized":"2024-09-13T14:45:20.390392+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":770,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344536+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:45:10.344536+0000","last_peered":"2024-09-13T14:45:10.344536+0000","last_clean":"2024-09-13T14:45:10.344536+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:45:10.344536+0000","last_undegraded":"2024-09-13T14:45:10.344536+0000","last_fullsized":"2024-09-13T14:45:10.344536+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387895+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:45:10.387895+0000","last_peered":"2024-09-13T14:45:10.387895+0000","last_clean":"2024-09-13T14:45:10.387895+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:45:10.387895+0000","last_undegraded":"2024-09-13T14:45:10.387895+0000","last_fullsized":"2024-09-13T14:45:10.387895+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388174+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:45:10.388174+0000","last_peered":"2024-09-13T14:45:10.388174+0000","last_clean":"2024-09-13T14:45:10.388174+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:45:10.388174+0000","last_undegraded":"2024-09-13T14:45:10.388174+0000","last_fullsized":"2024-09-13T14:45:10.388174+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":61,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388006+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:45:10.388006+0000","last_peered":"2024-09-13T14:45:10.388006+0000","last_clean":"2024-09-13T14:45:10.388006+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:45:10.388006+0000","last_undegraded":"2024-09-13T14:45:10.388006+0000","last_fullsized":"2024-09-13T14:45:10.388006+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":529,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542132+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:45:34.542132+0000","last_peered":"2024-09-13T14:45:34.542132+0000","last_clean":"2024-09-13T14:45:34.542132+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:45:34.542132+0000","last_undegraded":"2024-09-13T14:45:34.542132+0000","last_fullsized":"2024-09-13T14:45:34.542132+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":351,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.522896+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:45:10.522896+0000","last_peered":"2024-09-13T14:45:10.522896+0000","last_clean":"2024-09-13T14:45:10.522896+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:45:10.522896+0000","last_undegraded":"2024-09-13T14:45:10.522896+0000","last_fullsized":"2024-09-13T14:45:10.522896+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344612+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:45:10.344612+0000","last_peered":"2024-09-13T14:45:10.344612+0000","last_clean":"2024-09-13T14:45:10.344612+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:45:10.344612+0000","last_undegraded":"2024-09-13T14:45:10.344612+0000","last_fullsized":"2024-09-13T14:45:10.344612+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":534,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.532565+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:45:34.532565+0000","last_peered":"2024-09-13T14:45:34.532565+0000","last_clean":"2024-09-13T14:45:34.532565+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:45:34.532565+0000","last_undegraded":"2024-09-13T14:45:34.532565+0000","last_fullsized":"2024-09-13T14:45:34.532565+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":583,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390178+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:45:20.390178+0000","last_peered":"2024-09-13T14:45:20.390178+0000","last_clean":"2024-09-13T14:45:20.390178+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:45:20.390178+0000","last_undegraded":"2024-09-13T14:45:20.390178+0000","last_fullsized":"2024-09-13T14:45:20.390178+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":590,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542067+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:45:34.542067+0000","last_peered":"2024-09-13T14:45:34.542067+0000","last_clean":"2024-09-13T14:45:34.542067+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:45:34.542067+0000","last_undegraded":"2024-09-13T14:45:34.542067+0000","last_fullsized":"2024-09-13T14:45:34.542067+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":349,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344459+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:45:10.344459+0000","last_peered":"2024-09-13T14:45:10.344459+0000","last_clean":"2024-09-13T14:45:10.344459+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:45:10.344459+0000","last_undegraded":"2024-09-13T14:45:10.344459+0000","last_fullsized":"2024-09-13T14:45:10.344459+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":607,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542216+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:45:34.542216+0000","last_peered":"2024-09-13T14:45:34.542216+0000","last_clean":"2024-09-13T14:45:34.542216+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:45:34.542216+0000","last_undegraded":"2024-09-13T14:45:34.542216+0000","last_fullsized":"2024-09-13T14:45:34.542216+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":575,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390500+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:45:20.390500+0000","last_peered":"2024-09-13T14:45:20.390500+0000","last_clean":"2024-09-13T14:45:20.390500+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:45:20.390500+0000","last_undegraded":"2024-09-13T14:45:20.390500+0000","last_fullsized":"2024-09-13T14:45:20.390500+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":554,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.537488+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:45:34.537488+0000","last_peered":"2024-09-13T14:45:34.537488+0000","last_clean":"2024-09-13T14:45:34.537488+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:45:34.537488+0000","last_undegraded":"2024-09-13T14:45:34.537488+0000","last_fullsized":"2024-09-13T14:45:34.537488+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":603,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.533297+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:45:34.533297+0000","last_peered":"2024-09-13T14:45:34.533297+0000","last_clean":"2024-09-13T14:45:34.533297+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:45:34.533297+0000","last_undegraded":"2024-09-13T14:45:34.533297+0000","last_fullsized":"2024-09-13T14:45:34.533297+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.394468+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:45:10.394468+0000","last_peered":"2024-09-13T14:45:10.394468+0000","last_clean":"2024-09-13T14:45:10.394468+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:45:10.394468+0000","last_undegraded":"2024-09-13T14:45:10.394468+0000","last_fullsized":"2024-09-13T14:45:10.394468+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":500,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.536851+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:45:34.536851+0000","last_peered":"2024-09-13T14:45:34.536851+0000","last_clean":"2024-09-13T14:45:34.536851+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:45:34.536851+0000","last_undegraded":"2024-09-13T14:45:34.536851+0000","last_fullsized":"2024-09-13T14:45:34.536851+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":579,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390022+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:45:20.390022+0000","last_peered":"2024-09-13T14:45:20.390022+0000","last_clean":"2024-09-13T14:45:20.390022+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:45:20.390022+0000","last_undegraded":"2024-09-13T14:45:20.390022+0000","last_fullsized":"2024-09-13T14:45:20.390022+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.388058+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:45:10.388058+0000","last_peered":"2024-09-13T14:45:10.388058+0000","last_clean":"2024-09-13T14:45:10.388058+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:45:10.388058+0000","last_undegraded":"2024-09-13T14:45:10.388058+0000","last_fullsized":"2024-09-13T14:45:10.388058+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":317,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344870+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:45:10.344870+0000","last_peered":"2024-09-13T14:45:10.344870+0000","last_clean":"2024-09-13T14:45:10.344870+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:45:10.344870+0000","last_undegraded":"2024-09-13T14:45:10.344870+0000","last_fullsized":"2024-09-13T14:45:10.344870+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.394419+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:45:10.394419+0000","last_peered":"2024-09-13T14:45:10.394419+0000","last_clean":"2024-09-13T14:45:10.394419+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:45:10.394419+0000","last_undegraded":"2024-09-13T14:45:10.394419+0000","last_fullsized":"2024-09-13T14:45:10.394419+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":283,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.344535+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:45:10.344535+0000","last_peered":"2024-09-13T14:45:10.344535+0000","last_clean":"2024-09-13T14:45:10.344535+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:45:10.344535+0000","last_undegraded":"2024-09-13T14:45:10.344535+0000","last_fullsized":"2024-09-13T14:45:10.344535+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":599,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.532930+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:45:34.532930+0000","last_peered":"2024-09-13T14:45:34.532930+0000","last_clean":"2024-09-13T14:45:34.532930+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:45:34.532930+0000","last_undegraded":"2024-09-13T14:45:34.532930+0000","last_fullsized":"2024-09-13T14:45:34.532930+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":586,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.533712+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:45:34.533712+0000","last_peered":"2024-09-13T14:45:34.533712+0000","last_clean":"2024-09-13T14:45:34.533712+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:45:34.533712+0000","last_undegraded":"2024-09-13T14:45:34.533712+0000","last_fullsized":"2024-09-13T14:45:34.533712+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":51,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-09-13T14:45:10.387750+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:45:10.387750+0000","last_peered":"2024-09-13T14:45:10.387750+0000","last_clean":"2024-09-13T14:45:10.387750+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:45:10.387750+0000","last_undegraded":"2024-09-13T14:45:10.387750+0000","last_fullsized":"2024-09-13T14:45:10.387750+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":589,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-09-13T14:45:20.390010+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:45:20.390010+0000","last_peered":"2024-09-13T14:45:20.390010+0000","last_clean":"2024-09-13T14:45:20.390010+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:45:20.390010+0000","last_undegraded":"2024-09-13T14:45:20.390010+0000","last_fullsized":"2024-09-13T14:45:20.390010+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3799318552,"num_objects":57974,"num_object_clones":0,"num_object_copies":173922,"num_objects_missing_on_primary":2594,"num_objects_missing":2594,"num_objects_degraded":36736,"num_objects_misplaced":3547,"num_objects_unfound":0,"num_objects_dirty":57974,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124704,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23964,"num_bytes_recovered":1510080512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10500059136,"data_stored":10500046920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120304,"ondisk_log_size":120304,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397913,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2078844,"kb_used_data":1182836,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":91666308,"statfs":{"total":95995035648,"available":93866299392,"internally_reserved":0,"allocated":1211224064,"data_stored":1209368961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594141,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2118696,"kb_used_data":1524640,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91626456,"statfs":{"total":95995035648,"available":93825490944,"internally_reserved":0,"allocated":1561231360,"data_stored":1559396011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665595,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615080,"kb_used_data":282528,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93130072,"statfs":{"total":95995035648,"available":95365193728,"internally_reserved":0,"allocated":289308672,"data_stored":287469654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986597,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1742260,"kb_used_data":842412,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92002892,"statfs":{"total":95995035648,"available":94210961408,"internally_reserved":0,"allocated":862629888,"data_stored":860782883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697157,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2520056,"kb_used_data":1766060,"kb_used_omap":4,"kb_used_meta":753979,"kb_avail":91225096,"statfs":{"total":95995035648,"available":93414498304,"internally_reserved":0,"allocated":1808445440,"data_stored":1806602728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":772074571},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860684,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299196,"kb_used_data":1682100,"kb_used_omap":8,"kb_used_meta":617079,"kb_avail":91445956,"statfs":{"total":95995035648,"available":93640658944,"internally_reserved":0,"allocated":1722470400,"data_stored":1720620034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":631889740},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020355,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1938100,"kb_used_data":1630376,"kb_used_omap":9,"kb_used_meta":307702,"kb_avail":91807052,"statfs":{"total":95995035648,"available":94010421248,"internally_reserved":0,"allocated":1669505024,"data_stored":1667665785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":315087685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187745,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022752,"kb_used_data":1403928,"kb_used_omap":7,"kb_used_meta":618808,"kb_avail":91722400,"statfs":{"total":95995035648,"available":93923737600,"internally_reserved":0,"allocated":1437622272,"data_stored":1435791146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":633659723},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1426849792,"data_stored":1426849792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1660485632,"data_stored":1660485632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1714622464,"data_stored":1714618392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1801846784,"data_stored":1801846784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":857669632,"data_stored":857669632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":283185152,"data_stored":283181080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1556545536,"data_stored":1556545536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1198854144,"data_stored":1198850072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:39.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:45:39.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:39.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:40.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:40.119 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:40.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:40.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:40.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:40.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:40.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:40.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:40.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:40.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:40.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:40.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:40 smithi017 ceph-mon[103001]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:40 smithi017 ceph-mon[103001]: pgmap v1098: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36736/173313 objects degraded (21.196%); 3547/173313 objects misplaced (2.047%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:40 smithi017 ceph-mon[107556]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:40 smithi017 ceph-mon[107556]: pgmap v1098: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36736/173313 objects degraded (21.196%); 3547/173313 objects misplaced (2.047%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:40.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:40.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:40.811 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:40 smithi189 ceph-mon[90992]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:40.811 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:40 smithi189 ceph-mon[90992]: pgmap v1098: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36736/173313 objects degraded (21.196%); 3547/173313 objects misplaced (2.047%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:40.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:40.883 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:40.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:40.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:41.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:41.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:41.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:41.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:41.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:41.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:41.282 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:45:41.282 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190271 7.d 7:bc6c9de1:::benchmark_data_smithi017_220393_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 450.24246717199998, 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33251491900000002, 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.283 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190271 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:45:41.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036801+0000", 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036809+0000", 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8369999999999997e-06 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036827+0000", 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8233999999999999e-05 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036835+0000", 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036859+0000", 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3932999999999999e-05 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036943+0000", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4275000000000002e-05 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.037020+0000", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6403999999999996e-05 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.039717+0000", 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026977440000000002 2024-09-13T14:45:41.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045560+0000", 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058428489999999998 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369245+0000", 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32368499099999998 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369277+0000", 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1498999999999998e-05 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369317+0000", 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0759000000000001e-05 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.289 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190282 7.d 7:bc4d0eee:::benchmark_data_smithi017_220393_object71148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 450.23346067199998, 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32360401599999999, 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190282 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045808+0000", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045813+0000", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4319999999999998e-06 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045831+0000", 2024-09-13T14:45:41.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7847e-05 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045837+0000", 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3990000000000001e-06 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045875+0000", 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7481000000000002e-05 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045984+0000", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108773 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.046540+0000", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00055675799999999997 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.050112+0000", 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035716860000000001 2024-09-13T14:45:41.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.055205+0000", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0050931359999999998 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369356+0000", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31415043199999998 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369382+0000", 2024-09-13T14:45:41.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6463000000000001e-05 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369413+0000", 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0812999999999998e-05 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190303 7.d 7:b1faf55c:::benchmark_data_smithi017_220393_object71169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:45:41.296 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 450.197104529, 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28739342899999998, 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190303 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082164+0000", 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082170+0000", 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2580000000000001e-06 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082183+0000", 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3101e-05 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082190+0000", 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9580000000000001e-06 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082232+0000", 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2157000000000003e-05 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082315+0000", 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2902e-05 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082401+0000", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5847999999999995e-05 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085041+0000", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026398659999999998 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085709+0000", 2024-09-13T14:45:41.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00066763599999999999 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369484+0000", 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28377533599999999 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369521+0000", 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7574999999999999e-05 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369559+0000", 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7100000000000001e-05 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.303 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195365 7.7 7:e23819ca:::benchmark_data_smithi017_220393_object76231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 442.963551392, 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.15216712900000001, 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195365 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.304 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315717+0000", 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:41.305 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315726+0000", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8640000000000008e-06 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315738+0000", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1396999999999999e-05 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315743+0000", 2024-09-13T14:45:41.306 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5969999999999999e-06 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315786+0000", 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2601000000000001e-05 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315917+0000", 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000131326 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:45:41.307 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.316048+0000", 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013117099999999999 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.319756+0000", 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003707307 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.428425+0000", 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.108669617 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.308 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467786+0000", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039360456000000002 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467847+0000", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0847999999999997e-05 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467885+0000", 2024-09-13T14:45:41.309 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8680000000000002e-05 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195370 7.d 7:b9f15733:::benchmark_data_smithi017_220393_object76236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 442.96059437700001, 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14708705, 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.310 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195370 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.311 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318674+0000", 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318680+0000", 2024-09-13T14:45:41.312 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0700000000000003e-06 2024-09-13T14:45:41.313 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.313 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.313 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318691+0000", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0502e-05 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318695+0000", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2270000000000004e-06 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318730+0000", 2024-09-13T14:45:41.314 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5132999999999997e-05 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318848+0000", 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118114 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318955+0000", 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000106533 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.315 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.321687+0000", 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027322470000000001 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322829+0000", 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001141853 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465673+0000", 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14284481299999999 2024-09-13T14:45:41.316 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465722+0000", 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8504000000000002e-05 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465762+0000", 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0374999999999999e-05 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.317 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199290 7.7 7:e5b5f326:::benchmark_data_smithi017_220393_object80156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 437.79002209800001, 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20093702499999999, 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199290 2024-09-13T14:45:41.318 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489246+0000", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489252+0000", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7209999999999999e-06 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489264+0000", 2024-09-13T14:45:41.320 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2028000000000001e-05 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489270+0000", 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6219999999999997e-06 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489300+0000", 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0456000000000001e-05 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.321 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489418+0000", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117551 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489507+0000", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9654999999999995e-05 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.492597+0000", 2024-09-13T14:45:41.322 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030899809999999999 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.499941+0000", 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007343848 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690098+0000", 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190156834 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.323 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690145+0000", 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7094e-05 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690185+0000", 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9473e-05 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.324 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199306 7.d 7:b9985ddb:::benchmark_data_smithi017_220393_object80172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 437.77227797500001, 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.17325623900000001, 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199306 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.325 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506991+0000", 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:41.326 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506996+0000", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0529999999999997e-06 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507007+0000", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1533e-05 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507013+0000", 2024-09-13T14:45:41.327 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8640000000000001e-06 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507057+0000", 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4322e-05 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507173+0000", 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011561 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.328 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507291+0000", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117555 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.510005+0000", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027142020000000002 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.511252+0000", 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001246788 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.329 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680183+0000", 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.16893149399999999 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680220+0000", 2024-09-13T14:45:41.330 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7271999999999998e-05 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680248+0000", 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7537000000000001e-05 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200609 7.d 7:baf945be:::benchmark_data_smithi017_220393_object81475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 435.85167666400002, 2024-09-13T14:45:41.331 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34117809199999999, 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200609 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:45:41.332 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427592+0000", 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.333 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427598+0000", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1979999999999998e-06 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427610+0000", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.185e-05 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427615+0000", 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.772e-06 2024-09-13T14:45:41.334 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427649+0000", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3343999999999998e-05 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427739+0000", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0749999999999997e-05 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427862+0000", 2024-09-13T14:45:41.335 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012259799999999999 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434044+0000", 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061822839999999997 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435398+0000", 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001354035 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.336 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768686+0000", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33328797199999999 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768733+0000", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7101999999999997e-05 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768771+0000", 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7586000000000002e-05 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.337 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200614 7.d 7:bd88b3c8:::benchmark_data_smithi017_220393_object81480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 435.850697599, 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34032926800000002, 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200614 2024-09-13T14:45:41.338 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.339 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428571+0000", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428576+0000", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4469999999999999e-06 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428585+0000", 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:45:41.340 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428588+0000", 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5880000000000002e-06 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.430963+0000", 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023746639999999999 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.341 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431063+0000", 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000100412 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431173+0000", 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000109085 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434158+0000", 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029855020000000001 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.342 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435507+0000", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001348471 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768816+0000", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.333309466 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768866+0000", 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9571999999999999e-05 2024-09-13T14:45:41.343 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768901+0000", 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5741000000000002e-05 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200631 7.d 7:b540f86c:::benchmark_data_smithi017_220393_object81497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 435.71232512799997, 2024-09-13T14:45:41.344 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.202045216, 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200631 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.345 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566943+0000", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566949+0000", 2024-09-13T14:45:41.346 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4589999999999997e-06 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566960+0000", 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1046999999999999e-05 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566966+0000", 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9630000000000003e-06 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.347 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567008+0000", 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1878999999999997e-05 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567095+0000", 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7145000000000007e-05 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567169+0000", 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3732000000000005e-05 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.348 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.570791+0000", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036220850000000001 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.571627+0000", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000835928 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768940+0000", 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19731306200000001 2024-09-13T14:45:41.349 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768970+0000", 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0368000000000001e-05 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768990+0000", 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9626000000000001e-05 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.350 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200634 7.d 7:b80f92f6:::benchmark_data_smithi017_220393_object81500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 435.70667501600002, 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19650853099999999, 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200634 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.351 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572594+0000", 2024-09-13T14:45:41.352 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572600+0000", 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1920000000000002e-06 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572611+0000", 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1569999999999999e-05 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.353 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572616+0000", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2679999999999996e-06 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572656+0000", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0741000000000002e-05 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572783+0000", 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012668299999999999 2024-09-13T14:45:41.354 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572938+0000", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000155351 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.575602+0000", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026634129999999999 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.577024+0000", 2024-09-13T14:45:41.355 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014224999999999999 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769023+0000", 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19199869899999999 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769068+0000", 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5064000000000001e-05 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.356 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769103+0000", 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5173999999999999e-05 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.357 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200637 7.d 7:b4355551:::benchmark_data_smithi017_220393_object81503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 435.69765989299998, 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190526902, 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200637 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.358 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581609+0000", 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.359 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581614+0000", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9139999999999996e-06 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581626+0000", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1681999999999999e-05 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581632+0000", 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7780000000000004e-06 2024-09-13T14:45:41.360 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581664+0000", 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1585999999999999e-05 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581742+0000", 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8591999999999996e-05 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:45:41.361 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581828+0000", 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5816000000000006e-05 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584440+0000", 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026124199999999998 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584798+0000", 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035729199999999999 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.362 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772042+0000", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18724463299999999 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772092+0000", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9638000000000001e-05 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772137+0000", 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4747999999999999e-05 2024-09-13T14:45:41.363 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89676 8.9 8:94dade21:::benchmark_data_smithi017_253885_object89675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 297.58298774500003, 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2982149540000001, 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.364 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89676 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.365 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696280+0000", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696320+0000", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9366000000000002e-05 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696398+0000", 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8026000000000001e-05 2024-09-13T14:45:41.366 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696404+0000", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3029999999999996e-06 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.700853+0000", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0044487659999999998 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.700886+0000", 2024-09-13T14:45:41.367 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2885000000000001e-05 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693311+0000", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99242506200000002 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693320+0000", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2830000000000007e-06 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968082+0000", 2024-09-13T14:45:41.368 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2747621740000001 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968412+0000", 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032966499999999998 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968877+0000", 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00046530200000000001 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.369 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.973656+0000", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047786579999999999 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.978450+0000", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047938470000000004 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.990076+0000", 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011626645999999999 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.370 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.990261+0000", 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018436900000000001 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994497+0000", 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042362190000000003 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89677 8.9 8:975431c6:::benchmark_data_smithi017_253885_object89676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:45:41.371 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 297.57717430600002, 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3349859369999999, 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89677 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.372 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702094+0000", 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.373 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702164+0000", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0179999999999996e-05 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702176+0000", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2001000000000001e-05 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702182+0000", 2024-09-13T14:45:41.374 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6159999999999998e-06 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702225+0000", 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2599000000000003e-05 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702234+0000", 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6570000000000003e-06 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.375 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693344+0000", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.991110088 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693370+0000", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6111000000000001e-05 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.978555+0000", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.285184833 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.376 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.981496+0000", 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029411260000000001 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.981695+0000", 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019939900000000001 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994573+0000", 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012878069000000001 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.377 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994586+0000", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2709e-05 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032979+0000", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.03839298 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.033061+0000", 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1508000000000006e-05 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.378 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037081+0000", 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040207089999999999 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89693 8.9 8:9608d7fe:::benchmark_data_smithi017_253885_object89692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 297.54325978100002, 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3047559849999999, 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.379 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89693 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.380 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736008+0000", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736066+0000", 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7924999999999999e-05 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.381 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736083+0000", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7312e-05 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736092+0000", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1139999999999995e-06 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736158+0000", 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6256999999999999e-05 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.382 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736170+0000", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1457000000000001e-05 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693394+0000", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95722398200000003 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693401+0000", 2024-09-13T14:45:41.383 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0899999999999999e-06 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986096+0000", 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2926953990000001 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986273+0000", 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000176534 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:45:41.384 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986472+0000", 2024-09-13T14:45:41.385 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019907599999999999 2024-09-13T14:45:41.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994627+0000", 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081552020000000003 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.386 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994633+0000", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0209999999999996e-06 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037134+0000", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.042501078999999997 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037191+0000", 2024-09-13T14:45:41.387 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6669000000000002e-05 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.040766+0000", 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035749890000000002 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94271 8.5 8:a820db60:::benchmark_data_smithi017_253885_object2930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:45:41.388 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 253.96615521499999, 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96635259200000001, 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94271 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.389 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313112+0000", 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.390 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313121+0000", 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9709999999999993e-06 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313138+0000", 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6986999999999998e-05 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313145+0000", 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.391 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313203+0000", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7948000000000002e-05 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313215+0000", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2162e-05 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270121+0000", 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95690628200000005 2024-09-13T14:45:41.392 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270239+0000", 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011790699999999999 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270312+0000", 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3297e-05 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.393 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273683+0000", 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003371069 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273724+0000", 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0321000000000002e-05 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279374+0000", 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0056504349999999997 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.394 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279426+0000", 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.219e-05 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279467+0000", 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0595e-05 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94273 8.5 8:a6399fa1:::benchmark_data_smithi017_253885_object2932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:45:41.395 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 253.96075726000001, 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96120069299999999, 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94273 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.396 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318511+0000", 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.397 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318519+0000", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2490000000000001e-06 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318534+0000", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5048999999999999e-05 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318541+0000", 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.63e-06 2024-09-13T14:45:41.398 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318588+0000", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6711000000000001e-05 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318596+0000", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8810000000000002e-06 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273751+0000", 2024-09-13T14:45:41.399 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95515427799999997 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273869+0000", 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011859 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273946+0000", 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6808000000000001e-05 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.400 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279513+0000", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055664260000000002 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279564+0000", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1785e-05 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279640+0000", 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5766999999999995e-05 2024-09-13T14:45:41.401 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279681+0000", 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0914000000000002e-05 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279713+0000", 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2007e-05 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.402 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94300 8.5 8:ab0bc9a3:::benchmark_data_smithi017_253885_object2959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 253.58082588100001, 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.58689132899999996, 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94300 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.403 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698441+0000", 2024-09-13T14:45:41.404 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698452+0000", 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0397000000000001e-05 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698469+0000", 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6835000000000001e-05 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.405 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698477+0000", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7150000000000004e-06 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698526+0000", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8928000000000003e-05 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698536+0000", 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4839999999999993e-06 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.406 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.276872+0000", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.57833652099999999 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.276995+0000", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012297799999999999 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.277112+0000", 2024-09-13T14:45:41.407 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116341 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285181+0000", 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080698880000000008 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285217+0000", 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5676000000000002e-05 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.408 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285264+0000", 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6406000000000003e-05 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285295+0000", 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1965999999999999e-05 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285335+0000", 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9610999999999997e-05 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.409 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94316 8.5 8:a8a2bf9f:::benchmark_data_smithi017_253885_object2975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 253.405403017, 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.41461699099999999, 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94316 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.410 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873865+0000", 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.411 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873874+0000", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4609999999999997e-06 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873892+0000", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8150000000000001e-05 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873899+0000", 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3910000000000003e-06 2024-09-13T14:45:41.412 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873956+0000", 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7812999999999997e-05 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873969+0000", 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2099999999999999e-05 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.413 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279747+0000", 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40577885899999999 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279852+0000", 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000104879 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279944+0000", 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1917999999999996e-05 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.414 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285224+0000", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0052794679999999998 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285381+0000", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000157396 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288403+0000", 2024-09-13T14:45:41.415 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030220099999999999 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288443+0000", 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9412999999999997e-05 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288484+0000", 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1088999999999999e-05 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.416 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94332 8.5 8:aaf8da73:::benchmark_data_smithi017_253885_object2991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 253.177067287, 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18637316300000001, 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94332 2024-09-13T14:45:41.417 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.418 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102200+0000", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102210+0000", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0145e-05 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102228+0000", 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8025000000000001e-05 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.419 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102234+0000", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.139e-06 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102290+0000", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6217000000000001e-05 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102303+0000", 2024-09-13T14:45:41.420 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2911000000000001e-05 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282234+0000", 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.179931069 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282310+0000", 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5677000000000001e-05 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.421 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282375+0000", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5114e-05 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285415+0000", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030395769999999999 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288351+0000", 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029367970000000001 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.422 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288525+0000", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017392800000000001 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288555+0000", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9176999999999999e-05 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288576+0000", 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0978000000000001e-05 2024-09-13T14:45:41.423 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.424 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.424 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.424 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:45:41.424 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:45:41.424 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:45:41.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:41.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:41.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:41.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:41.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:41.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:41.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:41.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:41.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:41.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:42.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:42.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:42.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:42.138 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:42.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:42.192 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:42.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:45:42.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:42.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:42.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:42.456 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:42.547 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:45:42.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:42.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:42.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:42 smithi189 ceph-mon[90992]: pgmap v1099: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36691/172347 objects degraded (21.289%); 3504/172347 objects misplaced (2.033%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:42.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:42.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:42.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[103001]: pgmap v1099: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36691/172347 objects degraded (21.289%); 3504/172347 objects misplaced (2.033%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[107556]: pgmap v1099: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 36691/172347 objects degraded (21.289%); 3504/172347 objects misplaced (2.033%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:42.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:42.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:43.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:43.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:43.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:43.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:43.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:43.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:43.482 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:43.607 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:43.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:43.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:43 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:43 smithi189 ceph-mon[90992]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:45:43.828 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:43.828 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 36603/171909 objects degraded (21.292%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1541,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":354,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":57303,"data_bytes":3758197430,"bytes_used":15590920192,"bytes_avail":752369364992,"bytes_total":767960285184,"degraded_objects":36603,"degraded_total":171909,"degraded_ratio":0.21292078948746138,"misplaced_objects":3504,"misplaced_total":171909,"misplaced_ratio":0.02038287698724325,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":941561,"recovering_keys_per_sec":0,"num_objects_recovered":125,"num_bytes_recovered":8192000,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":122},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 30s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:45:43.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:43.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:43.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[103001]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:45:43.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:43.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/621238766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:43.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:43 smithi017 ceph-mon[107556]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:45:43.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:43.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:43.959 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:43.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:43.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:44.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:44.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:44.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:44.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:44.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:44.449 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:45:44.449 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:45:44.449 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:45:44.449 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:45:44.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:44.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:44.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[103001]: pgmap v1101: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36603/171909 objects degraded (21.292%); 3504/171909 objects misplaced (2.038%); 919 KiB/s, 14 objects/s recovering 2024-09-13T14:45:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36603/171909 objects degraded (21.292%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2144776803' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[107556]: pgmap v1101: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36603/171909 objects degraded (21.292%); 3504/171909 objects misplaced (2.038%); 919 KiB/s, 14 objects/s recovering 2024-09-13T14:45:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36603/171909 objects degraded (21.292%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2144776803' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:44.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:44.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:44.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:44 smithi189 ceph-mon[90992]: pgmap v1101: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 36603/171909 objects degraded (21.292%); 3504/171909 objects misplaced (2.038%); 919 KiB/s, 14 objects/s recovering 2024-09-13T14:45:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36603/171909 objects degraded (21.292%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2144776803' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:45:44.814 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:45:44.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:44.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:44.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:44.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:45.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:45.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:45.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:45.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:45.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:45.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:45.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:45.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:45.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:45.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:45.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:45.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:45.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:45.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:45.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:45.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:45.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:45.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:45:45.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:46.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:46.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:46.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:46.204 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:46.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:46.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:46.335 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:45:46.335 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:46.352 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:45:46.352 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:46.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:46.372 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:45:46.372 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:46.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:46.390 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:45:46.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:46.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:45:46.409 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:46.431 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:45:46.431 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:46.440 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:46.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:45:46.449 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:46.467 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:45:46.468 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:46.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:46.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:46.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:46.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:46.698 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:46 smithi189 ceph-mon[90992]: pgmap v1102: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 36560/171000 objects degraded (21.380%); 3460/171000 objects misplaced (2.023%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:46.698 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:46 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/253749810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:46 smithi017 ceph-mon[103001]: pgmap v1102: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 36560/171000 objects degraded (21.380%); 3460/171000 objects misplaced (2.023%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:46 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/253749810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:46 smithi017 ceph-mon[107556]: pgmap v1102: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 36560/171000 objects degraded (21.380%); 3460/171000 objects misplaced (2.023%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:45:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:46 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/253749810' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:46.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:46.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:46.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:46.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:46.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:47.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:47.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:47.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:47.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:47.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:47.358 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:47.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:47.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:47.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:47.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:47.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:47] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:45:47.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:47.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:47.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:47.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:47.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:48.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:48.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:48.072 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:48.072 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:48.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:48.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:48.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:45:48.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:48.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:48.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:48.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:48.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:48.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:48.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:48.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:48.712 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:48 smithi189 ceph-mon[90992]: pgmap v1103: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 36458/170307 objects degraded (21.407%); 3460/170307 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:48.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:45:48.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:48.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:48.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:48 smithi017 ceph-mon[103001]: pgmap v1103: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 36458/170307 objects degraded (21.407%); 3460/170307 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:48.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:48 smithi017 ceph-mon[107556]: pgmap v1103: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 36458/170307 objects degraded (21.407%); 3460/170307 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:48.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:48.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:48.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:48.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:49.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:49.072 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:45:49.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:49.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:49.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:49.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:49.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:49.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:49.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:49.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:49.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:49.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:45:49.579 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:45:49.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:49.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36458/170307 objects degraded (21.407%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36458/170307 objects degraded (21.407%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:49.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:49.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:49.883 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:49.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:49.980 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36458/170307 objects degraded (21.407%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:50.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:50.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.255 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:45:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:45:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:50.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:50.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:50.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:50.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:50.483 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1104,"stamp":"2024-09-13T14:45:49.229741+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3703278262,"num_objects":56465,"num_object_clones":0,"num_object_copies":169395,"num_objects_missing_on_primary":2506,"num_objects_missing":2506,"num_objects_degraded":36458,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":56465,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":127188,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24203,"num_bytes_recovered":1522174109,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122873,"ondisk_log_size":122873,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15115092,"kb_used_data":10074192,"kb_used_omap":44,"kb_used_meta":5040787,"kb_avail":734846124,"statfs":{"total":767960285184,"available":752482430976,"internally_reserved":0,"allocated":10315972608,"data_stored":10301130679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5161766135},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,2,2,0,2,6,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85590016,"num_objects":-1306,"num_object_clones":0,"num_object_copies":-3918,"num_objects_missing_on_primary":-88,"num_objects_missing":-88,"num_objects_degraded":-278,"num_objects_misplaced":-87,"num_objects_unfound":0,"num_objects_dirty":-1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1306,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002804"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":592,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.581829+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:45:42.581829+0000","last_peered":"2024-09-13T14:45:42.581829+0000","last_clean":"2024-09-13T14:45:42.581829+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:45:42.581829+0000","last_undegraded":"2024-09-13T14:45:42.581829+0000","last_fullsized":"2024-09-13T14:45:42.581829+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":352,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.580977+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:45:34.580977+0000","last_peered":"2024-09-13T14:45:34.580977+0000","last_clean":"2024-09-13T14:45:34.580977+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:45:34.580977+0000","last_undegraded":"2024-09-13T14:45:34.580977+0000","last_fullsized":"2024-09-13T14:45:34.580977+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":281,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528954+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:45:34.528954+0000","last_peered":"2024-09-13T14:45:34.528954+0000","last_clean":"2024-09-13T14:45:34.528954+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:45:34.528954+0000","last_undegraded":"2024-09-13T14:45:34.528954+0000","last_fullsized":"2024-09-13T14:45:34.528954+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":577,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.611722+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:45:42.611722+0000","last_peered":"2024-09-13T14:45:42.611722+0000","last_clean":"2024-09-13T14:45:42.611722+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:45:42.611722+0000","last_undegraded":"2024-09-13T14:45:42.611722+0000","last_fullsized":"2024-09-13T14:45:42.611722+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":578,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.613328+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:45:42.613328+0000","last_peered":"2024-09-13T14:45:42.613328+0000","last_clean":"2024-09-13T14:45:42.613328+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:45:42.613328+0000","last_undegraded":"2024-09-13T14:45:42.613328+0000","last_fullsized":"2024-09-13T14:45:42.613328+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":321,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.581165+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:45:34.581165+0000","last_peered":"2024-09-13T14:45:34.581165+0000","last_clean":"2024-09-13T14:45:34.581165+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:45:34.581165+0000","last_undegraded":"2024-09-13T14:45:34.581165+0000","last_fullsized":"2024-09-13T14:45:34.581165+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":592,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.582606+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:45:42.582606+0000","last_peered":"2024-09-13T14:45:42.582606+0000","last_clean":"2024-09-13T14:45:42.582606+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:45:42.582606+0000","last_undegraded":"2024-09-13T14:45:42.582606+0000","last_fullsized":"2024-09-13T14:45:42.582606+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":343,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.541712+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:45:34.541712+0000","last_peered":"2024-09-13T14:45:34.541712+0000","last_clean":"2024-09-13T14:45:34.541712+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:45:34.541712+0000","last_undegraded":"2024-09-13T14:45:34.541712+0000","last_fullsized":"2024-09-13T14:45:34.541712+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"354'7850","reported_seq":14577,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:47.052372+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:45:47.052372+0000","last_peered":"2024-09-13T14:45:47.052372+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:45:47.052372+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7850,"log_dups_size":0,"ondisk_log_size":7850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225443840,"num_objects":3440,"num_object_clones":0,"num_object_copies":10320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5793,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7850,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2206,"num_bytes_recovered":144572416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2455}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":347,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.581591+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:45:34.581591+0000","last_peered":"2024-09-13T14:45:34.581591+0000","last_clean":"2024-09-13T14:45:34.581591+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:45:34.581591+0000","last_undegraded":"2024-09-13T14:45:34.581591+0000","last_fullsized":"2024-09-13T14:45:34.581591+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528761+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:45:34.528761+0000","last_peered":"2024-09-13T14:45:34.528761+0000","last_clean":"2024-09-13T14:45:34.528761+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:45:34.528761+0000","last_undegraded":"2024-09-13T14:45:34.528761+0000","last_fullsized":"2024-09-13T14:45:34.528761+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"354'7913","reported_seq":17464,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:48.032086+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:45:48.032086+0000","last_peered":"2024-09-13T14:45:48.032086+0000","last_clean":"2024-09-13T14:45:48.032086+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:45:48.032086+0000","last_undegraded":"2024-09-13T14:45:48.032086+0000","last_fullsized":"2024-09-13T14:45:48.032086+0000","mapping_epoch":326,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234291200,"num_objects":3575,"num_object_clones":0,"num_object_copies":10725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7913,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":365,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.584223+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:45:34.584223+0000","last_peered":"2024-09-13T14:45:34.584223+0000","last_clean":"2024-09-13T14:45:34.584223+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:45:34.584223+0000","last_undegraded":"2024-09-13T14:45:34.584223+0000","last_fullsized":"2024-09-13T14:45:34.584223+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554492+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:45:34.554492+0000","last_peered":"2024-09-13T14:45:34.554492+0000","last_clean":"2024-09-13T14:45:34.554492+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:45:34.554492+0000","last_undegraded":"2024-09-13T14:45:34.554492+0000","last_fullsized":"2024-09-13T14:45:34.554492+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"354'8222","reported_seq":8337,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:48.904301+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:45:48.904301+0000","last_peered":"2024-09-13T14:45:48.904301+0000","last_clean":"2024-09-13T14:45:48.904301+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:45:48.904301+0000","last_undegraded":"2024-09-13T14:45:48.904301+0000","last_fullsized":"2024-09-13T14:45:48.904301+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8222,"log_dups_size":0,"ondisk_log_size":8222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234356736,"num_objects":3576,"num_object_clones":0,"num_object_copies":10728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8222,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":319,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.581732+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:45:34.581732+0000","last_peered":"2024-09-13T14:45:34.581732+0000","last_clean":"2024-09-13T14:45:34.581732+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:45:34.581732+0000","last_undegraded":"2024-09-13T14:45:34.581732+0000","last_fullsized":"2024-09-13T14:45:34.581732+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":334,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.541834+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:45:34.541834+0000","last_peered":"2024-09-13T14:45:34.541834+0000","last_clean":"2024-09-13T14:45:34.541834+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:45:34.541834+0000","last_undegraded":"2024-09-13T14:45:34.541834+0000","last_fullsized":"2024-09-13T14:45:34.541834+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"354'7976","reported_seq":8091,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:45.041223+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:45:45.041223+0000","last_peered":"2024-09-13T14:45:45.041223+0000","last_clean":"2024-09-13T14:45:45.041223+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:45:45.041223+0000","last_undegraded":"2024-09-13T14:45:45.041223+0000","last_fullsized":"2024-09-13T14:45:45.041223+0000","mapping_epoch":303,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7576,"log_dups_size":0,"ondisk_log_size":7576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233963520,"num_objects":3570,"num_object_clones":0,"num_object_copies":10710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7976,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":282,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.529052+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:45:34.529052+0000","last_peered":"2024-09-13T14:45:34.529052+0000","last_clean":"2024-09-13T14:45:34.529052+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:45:34.529052+0000","last_undegraded":"2024-09-13T14:45:34.529052+0000","last_fullsized":"2024-09-13T14:45:34.529052+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":603,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.612738+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:45:42.612738+0000","last_peered":"2024-09-13T14:45:42.612738+0000","last_clean":"2024-09-13T14:45:42.612738+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:45:42.612738+0000","last_undegraded":"2024-09-13T14:45:42.612738+0000","last_fullsized":"2024-09-13T14:45:42.612738+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"354'7911","reported_seq":8024,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:45.272536+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:45:45.272536+0000","last_peered":"2024-09-13T14:45:45.272536+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:45:45.272536+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232718336,"num_objects":3551,"num_object_clones":0,"num_object_copies":10653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7911,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2082}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":465,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.580458+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:45:34.580458+0000","last_peered":"2024-09-13T14:45:34.580458+0000","last_clean":"2024-09-13T14:45:34.580458+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:45:34.580458+0000","last_undegraded":"2024-09-13T14:45:34.580458+0000","last_fullsized":"2024-09-13T14:45:34.580458+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":601,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.581436+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:45:42.581436+0000","last_peered":"2024-09-13T14:45:42.581436+0000","last_clean":"2024-09-13T14:45:42.581436+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:45:42.581436+0000","last_undegraded":"2024-09-13T14:45:42.581436+0000","last_fullsized":"2024-09-13T14:45:42.581436+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554603+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:45:34.554603+0000","last_peered":"2024-09-13T14:45:34.554603+0000","last_clean":"2024-09-13T14:45:34.554603+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:45:34.554603+0000","last_undegraded":"2024-09-13T14:45:34.554603+0000","last_fullsized":"2024-09-13T14:45:34.554603+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"354'7608","reported_seq":7723,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:44.759280+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:45:44.759280+0000","last_peered":"2024-09-13T14:45:44.759280+0000","last_clean":"2024-09-13T14:45:44.759280+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:45:44.759280+0000","last_undegraded":"2024-09-13T14:45:44.759280+0000","last_fullsized":"2024-09-13T14:45:44.759280+0000","mapping_epoch":303,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228458496,"num_objects":3486,"num_object_clones":0,"num_object_copies":10458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7608,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":169,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528841+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:45:34.528841+0000","last_peered":"2024-09-13T14:45:34.528841+0000","last_clean":"2024-09-13T14:45:34.528841+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:45:34.528841+0000","last_undegraded":"2024-09-13T14:45:34.528841+0000","last_fullsized":"2024-09-13T14:45:34.528841+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":355,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.541739+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:45:34.541739+0000","last_peered":"2024-09-13T14:45:34.541739+0000","last_clean":"2024-09-13T14:45:34.541739+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:45:34.541739+0000","last_undegraded":"2024-09-13T14:45:34.541739+0000","last_fullsized":"2024-09-13T14:45:34.541739+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":67,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554868+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:45:34.554868+0000","last_peered":"2024-09-13T14:45:34.554868+0000","last_clean":"2024-09-13T14:45:34.554868+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:45:34.554868+0000","last_undegraded":"2024-09-13T14:45:34.554868+0000","last_fullsized":"2024-09-13T14:45:34.554868+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"354'7849","reported_seq":13994,"reported_epoch":354,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:44.471453+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:45:44.471453+0000","last_peered":"2024-09-13T14:45:44.471453+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:45:44.471453+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7349,"log_dups_size":0,"ondisk_log_size":7349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":2506,"num_objects_missing":2506,"num_objects_degraded":2881,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7849,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2143,"num_bytes_recovered":140443648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2506},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554341+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:45:34.554341+0000","last_peered":"2024-09-13T14:45:34.554341+0000","last_clean":"2024-09-13T14:45:34.554341+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:45:34.554341+0000","last_undegraded":"2024-09-13T14:45:34.554341+0000","last_fullsized":"2024-09-13T14:45:34.554341+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.557238+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:45:34.557238+0000","last_peered":"2024-09-13T14:45:34.557238+0000","last_clean":"2024-09-13T14:45:34.557238+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:45:34.557238+0000","last_undegraded":"2024-09-13T14:45:34.557238+0000","last_fullsized":"2024-09-13T14:45:34.557238+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"354'7686","reported_seq":7801,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:44.973176+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:45:44.973176+0000","last_peered":"2024-09-13T14:45:44.973176+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:45:44.973176+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227540992,"num_objects":3472,"num_object_clones":0,"num_object_copies":10416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7686,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-1994}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":162,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528833+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:45:34.528833+0000","last_peered":"2024-09-13T14:45:34.528833+0000","last_clean":"2024-09-13T14:45:34.528833+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:45:34.528833+0000","last_undegraded":"2024-09-13T14:45:34.528833+0000","last_fullsized":"2024-09-13T14:45:34.528833+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":594,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.588102+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:45:42.588102+0000","last_peered":"2024-09-13T14:45:42.588102+0000","last_clean":"2024-09-13T14:45:42.588102+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:45:42.588102+0000","last_undegraded":"2024-09-13T14:45:42.588102+0000","last_fullsized":"2024-09-13T14:45:42.588102+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":555,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548987+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:45:42.548987+0000","last_peered":"2024-09-13T14:45:42.548987+0000","last_clean":"2024-09-13T14:45:42.548987+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:45:42.548987+0000","last_undegraded":"2024-09-13T14:45:42.548987+0000","last_fullsized":"2024-09-13T14:45:42.548987+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"354'7763","reported_seq":7829,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:47.938341+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:45:47.938341+0000","last_peered":"2024-09-13T14:45:47.938341+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:45:47.938341+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7563,"log_dups_size":0,"ondisk_log_size":7563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226557952,"num_objects":3457,"num_object_clones":0,"num_object_copies":10371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7763,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2039}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":456,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.580555+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:45:34.580555+0000","last_peered":"2024-09-13T14:45:34.580555+0000","last_clean":"2024-09-13T14:45:34.580555+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:45:34.580555+0000","last_undegraded":"2024-09-13T14:45:34.580555+0000","last_fullsized":"2024-09-13T14:45:34.580555+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":878,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.534802+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:45:34.534802+0000","last_peered":"2024-09-13T14:45:34.534802+0000","last_clean":"2024-09-13T14:45:34.534802+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:45:34.534802+0000","last_undegraded":"2024-09-13T14:45:34.534802+0000","last_fullsized":"2024-09-13T14:45:34.534802+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":547,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.580032+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:45:42.580032+0000","last_peered":"2024-09-13T14:45:42.580032+0000","last_clean":"2024-09-13T14:45:42.580032+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:45:42.580032+0000","last_undegraded":"2024-09-13T14:45:42.580032+0000","last_fullsized":"2024-09-13T14:45:42.580032+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":320,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542152+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:45:34.542152+0000","last_peered":"2024-09-13T14:45:34.542152+0000","last_clean":"2024-09-13T14:45:34.542152+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:45:34.542152+0000","last_undegraded":"2024-09-13T14:45:34.542152+0000","last_fullsized":"2024-09-13T14:45:34.542152+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"354'8001","reported_seq":8114,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:45.020595+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:45:45.020595+0000","last_peered":"2024-09-13T14:45:45.020595+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:45:45.020595+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232456192,"num_objects":3547,"num_object_clones":0,"num_object_copies":10641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8001,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2133}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":401,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548136+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:45:42.548136+0000","last_peered":"2024-09-13T14:45:42.548136+0000","last_clean":"2024-09-13T14:45:42.548136+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:45:42.548136+0000","last_undegraded":"2024-09-13T14:45:42.548136+0000","last_fullsized":"2024-09-13T14:45:42.548136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":744,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.580552+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:45:34.580552+0000","last_peered":"2024-09-13T14:45:34.580552+0000","last_clean":"2024-09-13T14:45:34.580552+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:45:34.580552+0000","last_undegraded":"2024-09-13T14:45:34.580552+0000","last_fullsized":"2024-09-13T14:45:34.580552+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":596,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.615020+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:45:42.615020+0000","last_peered":"2024-09-13T14:45:42.615020+0000","last_clean":"2024-09-13T14:45:42.615020+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:45:42.615020+0000","last_undegraded":"2024-09-13T14:45:42.615020+0000","last_fullsized":"2024-09-13T14:45:42.615020+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":601,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.613479+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:45:42.613479+0000","last_peered":"2024-09-13T14:45:42.613479+0000","last_clean":"2024-09-13T14:45:42.613479+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:45:42.613479+0000","last_undegraded":"2024-09-13T14:45:42.613479+0000","last_fullsized":"2024-09-13T14:45:42.613479+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"354'7996","reported_seq":17652,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:45.550963+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:45:45.550963+0000","last_peered":"2024-09-13T14:45:45.550963+0000","last_clean":"2024-09-13T14:45:45.550963+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:45:45.550963+0000","last_undegraded":"2024-09-13T14:45:45.550963+0000","last_fullsized":"2024-09-13T14:45:45.550963+0000","mapping_epoch":330,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240386048,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7996,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":485,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.580667+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:45:34.580667+0000","last_peered":"2024-09-13T14:45:34.580667+0000","last_clean":"2024-09-13T14:45:34.580667+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:45:34.580667+0000","last_undegraded":"2024-09-13T14:45:34.580667+0000","last_fullsized":"2024-09-13T14:45:34.580667+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":565,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.580298+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:45:42.580298+0000","last_peered":"2024-09-13T14:45:42.580298+0000","last_clean":"2024-09-13T14:45:42.580298+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:45:42.580298+0000","last_undegraded":"2024-09-13T14:45:42.580298+0000","last_fullsized":"2024-09-13T14:45:42.580298+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":533,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.547939+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:45:42.547939+0000","last_peered":"2024-09-13T14:45:42.547939+0000","last_clean":"2024-09-13T14:45:42.547939+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:45:42.547939+0000","last_undegraded":"2024-09-13T14:45:42.547939+0000","last_fullsized":"2024-09-13T14:45:42.547939+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.529150+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:45:34.529150+0000","last_peered":"2024-09-13T14:45:34.529150+0000","last_clean":"2024-09-13T14:45:34.529150+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:45:34.529150+0000","last_undegraded":"2024-09-13T14:45:34.529150+0000","last_fullsized":"2024-09-13T14:45:34.529150+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"354'7699","reported_seq":7813,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:45.571506+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:45:45.571506+0000","last_peered":"2024-09-13T14:45:45.571506+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:45:45.571506+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231407616,"num_objects":3531,"num_object_clones":0,"num_object_copies":10593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7699,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1978}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":162,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528515+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:45:34.528515+0000","last_peered":"2024-09-13T14:45:34.528515+0000","last_clean":"2024-09-13T14:45:34.528515+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:45:34.528515+0000","last_undegraded":"2024-09-13T14:45:34.528515+0000","last_fullsized":"2024-09-13T14:45:34.528515+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":969,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528911+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:45:34.528911+0000","last_peered":"2024-09-13T14:45:34.528911+0000","last_clean":"2024-09-13T14:45:34.528911+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:45:34.528911+0000","last_undegraded":"2024-09-13T14:45:34.528911+0000","last_fullsized":"2024-09-13T14:45:34.528911+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.555355+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:45:34.555355+0000","last_peered":"2024-09-13T14:45:34.555355+0000","last_clean":"2024-09-13T14:45:34.555355+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:45:34.555355+0000","last_undegraded":"2024-09-13T14:45:34.555355+0000","last_fullsized":"2024-09-13T14:45:34.555355+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":554,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548158+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:45:42.548158+0000","last_peered":"2024-09-13T14:45:42.548158+0000","last_clean":"2024-09-13T14:45:42.548158+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:45:42.548158+0000","last_undegraded":"2024-09-13T14:45:42.548158+0000","last_fullsized":"2024-09-13T14:45:42.548158+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"354'7877","reported_seq":17272,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.014288+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:45:49.014288+0000","last_peered":"2024-09-13T14:45:49.014288+0000","last_clean":"2024-09-13T14:45:49.014288+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:45:49.014288+0000","last_undegraded":"2024-09-13T14:45:49.014288+0000","last_fullsized":"2024-09-13T14:45:49.014288+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7877,"log_dups_size":0,"ondisk_log_size":7877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7877,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":353,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.544259+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:45:34.544259+0000","last_peered":"2024-09-13T14:45:34.544259+0000","last_clean":"2024-09-13T14:45:34.544259+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:45:34.544259+0000","last_undegraded":"2024-09-13T14:45:34.544259+0000","last_fullsized":"2024-09-13T14:45:34.544259+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528842+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:45:34.528842+0000","last_peered":"2024-09-13T14:45:34.528842+0000","last_clean":"2024-09-13T14:45:34.528842+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:45:34.528842+0000","last_undegraded":"2024-09-13T14:45:34.528842+0000","last_fullsized":"2024-09-13T14:45:34.528842+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1133,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548329+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:45:42.548329+0000","last_peered":"2024-09-13T14:45:42.548329+0000","last_clean":"2024-09-13T14:45:42.548329+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:45:42.548329+0000","last_undegraded":"2024-09-13T14:45:42.548329+0000","last_fullsized":"2024-09-13T14:45:42.548329+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":535,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548366+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:45:42.548366+0000","last_peered":"2024-09-13T14:45:42.548366+0000","last_clean":"2024-09-13T14:45:42.548366+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:45:42.548366+0000","last_undegraded":"2024-09-13T14:45:42.548366+0000","last_fullsized":"2024-09-13T14:45:42.548366+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"354'7721","reported_seq":7836,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:45.365118+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:45:45.365118+0000","last_peered":"2024-09-13T14:45:45.365118+0000","last_clean":"2024-09-13T14:45:45.365118+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:45:45.365118+0000","last_undegraded":"2024-09-13T14:45:45.365118+0000","last_fullsized":"2024-09-13T14:45:45.365118+0000","mapping_epoch":303,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7721,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":353,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.542145+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:45:34.542145+0000","last_peered":"2024-09-13T14:45:34.542145+0000","last_clean":"2024-09-13T14:45:34.542145+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:45:34.542145+0000","last_undegraded":"2024-09-13T14:45:34.542145+0000","last_fullsized":"2024-09-13T14:45:34.542145+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":544,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548244+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:45:42.548244+0000","last_peered":"2024-09-13T14:45:42.548244+0000","last_clean":"2024-09-13T14:45:42.548244+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:45:42.548244+0000","last_undegraded":"2024-09-13T14:45:42.548244+0000","last_fullsized":"2024-09-13T14:45:42.548244+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":748,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.581753+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:45:34.581753+0000","last_peered":"2024-09-13T14:45:34.581753+0000","last_clean":"2024-09-13T14:45:34.581753+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:45:34.581753+0000","last_undegraded":"2024-09-13T14:45:34.581753+0000","last_fullsized":"2024-09-13T14:45:34.581753+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":579,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.612641+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:45:42.612641+0000","last_peered":"2024-09-13T14:45:42.612641+0000","last_clean":"2024-09-13T14:45:42.612641+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:45:42.612641+0000","last_undegraded":"2024-09-13T14:45:42.612641+0000","last_fullsized":"2024-09-13T14:45:42.612641+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"354'8128","reported_seq":8243,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:48.782033+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:45:48.782033+0000","last_peered":"2024-09-13T14:45:48.782033+0000","last_clean":"2024-09-13T14:45:48.782033+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:45:48.782033+0000","last_undegraded":"2024-09-13T14:45:48.782033+0000","last_fullsized":"2024-09-13T14:45:48.782033+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8128,"log_dups_size":0,"ondisk_log_size":8128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233439232,"num_objects":3562,"num_object_clones":0,"num_object_copies":10686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8128,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554783+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:45:34.554783+0000","last_peered":"2024-09-13T14:45:34.554783+0000","last_clean":"2024-09-13T14:45:34.554783+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:45:34.554783+0000","last_undegraded":"2024-09-13T14:45:34.554783+0000","last_fullsized":"2024-09-13T14:45:34.554783+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":580,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.582992+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:45:42.582992+0000","last_peered":"2024-09-13T14:45:42.582992+0000","last_clean":"2024-09-13T14:45:42.582992+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:45:42.582992+0000","last_undegraded":"2024-09-13T14:45:42.582992+0000","last_fullsized":"2024-09-13T14:45:42.582992+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":920,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548651+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:45:42.548651+0000","last_peered":"2024-09-13T14:45:42.548651+0000","last_clean":"2024-09-13T14:45:42.548651+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:45:42.548651+0000","last_undegraded":"2024-09-13T14:45:42.548651+0000","last_fullsized":"2024-09-13T14:45:42.548651+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":577,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.588000+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:45:42.588000+0000","last_peered":"2024-09-13T14:45:42.588000+0000","last_clean":"2024-09-13T14:45:42.588000+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:45:42.588000+0000","last_undegraded":"2024-09-13T14:45:42.588000+0000","last_fullsized":"2024-09-13T14:45:42.588000+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"354'8041","reported_seq":19406,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:47.707150+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:45:47.707150+0000","last_peered":"2024-09-13T14:45:47.707150+0000","last_clean":"2024-09-13T14:45:47.707150+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:45:47.707150+0000","last_undegraded":"2024-09-13T14:45:47.707150+0000","last_fullsized":"2024-09-13T14:45:47.707150+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8041,"log_dups_size":0,"ondisk_log_size":8041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227541016,"num_objects":3473,"num_object_clones":0,"num_object_copies":10419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3473,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8041,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":431,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.581638+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:45:42.581638+0000","last_peered":"2024-09-13T14:45:42.581638+0000","last_clean":"2024-09-13T14:45:42.581638+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:45:42.581638+0000","last_undegraded":"2024-09-13T14:45:42.581638+0000","last_fullsized":"2024-09-13T14:45:42.581638+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":615,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.613169+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:45:42.613169+0000","last_peered":"2024-09-13T14:45:42.613169+0000","last_clean":"2024-09-13T14:45:42.613169+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:45:42.613169+0000","last_undegraded":"2024-09-13T14:45:42.613169+0000","last_fullsized":"2024-09-13T14:45:42.613169+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":772,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.544335+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:45:34.544335+0000","last_peered":"2024-09-13T14:45:34.544335+0000","last_clean":"2024-09-13T14:45:34.544335+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:45:34.544335+0000","last_undegraded":"2024-09-13T14:45:34.544335+0000","last_fullsized":"2024-09-13T14:45:34.544335+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":67,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554415+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:45:34.554415+0000","last_peered":"2024-09-13T14:45:34.554415+0000","last_clean":"2024-09-13T14:45:34.554415+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:45:34.554415+0000","last_undegraded":"2024-09-13T14:45:34.554415+0000","last_fullsized":"2024-09-13T14:45:34.554415+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.555374+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:45:34.555374+0000","last_peered":"2024-09-13T14:45:34.555374+0000","last_clean":"2024-09-13T14:45:34.555374+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:45:34.555374+0000","last_undegraded":"2024-09-13T14:45:34.555374+0000","last_fullsized":"2024-09-13T14:45:34.555374+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":63,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554527+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:45:34.554527+0000","last_peered":"2024-09-13T14:45:34.554527+0000","last_clean":"2024-09-13T14:45:34.554527+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:45:34.554527+0000","last_undegraded":"2024-09-13T14:45:34.554527+0000","last_fullsized":"2024-09-13T14:45:34.554527+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":531,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548222+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:45:42.548222+0000","last_peered":"2024-09-13T14:45:42.548222+0000","last_clean":"2024-09-13T14:45:42.548222+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:45:42.548222+0000","last_undegraded":"2024-09-13T14:45:42.548222+0000","last_fullsized":"2024-09-13T14:45:42.548222+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":353,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.581605+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:45:34.581605+0000","last_peered":"2024-09-13T14:45:34.581605+0000","last_clean":"2024-09-13T14:45:34.581605+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:45:34.581605+0000","last_undegraded":"2024-09-13T14:45:34.581605+0000","last_fullsized":"2024-09-13T14:45:34.581605+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":319,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.529112+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:45:34.529112+0000","last_peered":"2024-09-13T14:45:34.529112+0000","last_clean":"2024-09-13T14:45:34.529112+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:45:34.529112+0000","last_undegraded":"2024-09-13T14:45:34.529112+0000","last_fullsized":"2024-09-13T14:45:34.529112+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":536,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.579633+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:45:42.579633+0000","last_peered":"2024-09-13T14:45:42.579633+0000","last_clean":"2024-09-13T14:45:42.579633+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:45:42.579633+0000","last_undegraded":"2024-09-13T14:45:42.579633+0000","last_fullsized":"2024-09-13T14:45:42.579633+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":588,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.612166+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:45:42.612166+0000","last_peered":"2024-09-13T14:45:42.612166+0000","last_clean":"2024-09-13T14:45:42.612166+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:45:42.612166+0000","last_undegraded":"2024-09-13T14:45:42.612166+0000","last_fullsized":"2024-09-13T14:45:42.612166+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":592,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.587941+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:45:42.587941+0000","last_peered":"2024-09-13T14:45:42.587941+0000","last_clean":"2024-09-13T14:45:42.587941+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:45:42.587941+0000","last_undegraded":"2024-09-13T14:45:42.587941+0000","last_fullsized":"2024-09-13T14:45:42.587941+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":351,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.544317+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:45:34.544317+0000","last_peered":"2024-09-13T14:45:34.544317+0000","last_clean":"2024-09-13T14:45:34.544317+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:45:34.544317+0000","last_undegraded":"2024-09-13T14:45:34.544317+0000","last_fullsized":"2024-09-13T14:45:34.544317+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":610,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.588075+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:45:42.588075+0000","last_peered":"2024-09-13T14:45:42.588075+0000","last_clean":"2024-09-13T14:45:42.588075+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:45:42.588075+0000","last_undegraded":"2024-09-13T14:45:42.588075+0000","last_fullsized":"2024-09-13T14:45:42.588075+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":580,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.613384+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:45:42.613384+0000","last_peered":"2024-09-13T14:45:42.613384+0000","last_clean":"2024-09-13T14:45:42.613384+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:45:42.613384+0000","last_undegraded":"2024-09-13T14:45:42.613384+0000","last_fullsized":"2024-09-13T14:45:42.613384+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":556,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.547965+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:45:42.547965+0000","last_peered":"2024-09-13T14:45:42.547965+0000","last_clean":"2024-09-13T14:45:42.547965+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:45:42.547965+0000","last_undegraded":"2024-09-13T14:45:42.547965+0000","last_fullsized":"2024-09-13T14:45:42.547965+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":605,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.580653+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:45:42.580653+0000","last_peered":"2024-09-13T14:45:42.580653+0000","last_clean":"2024-09-13T14:45:42.580653+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:45:42.580653+0000","last_undegraded":"2024-09-13T14:45:42.580653+0000","last_fullsized":"2024-09-13T14:45:42.580653+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":52,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.557270+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:45:34.557270+0000","last_peered":"2024-09-13T14:45:34.557270+0000","last_clean":"2024-09-13T14:45:34.557270+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:45:34.557270+0000","last_undegraded":"2024-09-13T14:45:34.557270+0000","last_fullsized":"2024-09-13T14:45:34.557270+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":502,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548552+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:45:42.548552+0000","last_peered":"2024-09-13T14:45:42.548552+0000","last_clean":"2024-09-13T14:45:42.548552+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:45:42.548552+0000","last_undegraded":"2024-09-13T14:45:42.548552+0000","last_fullsized":"2024-09-13T14:45:42.548552+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":583,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.612522+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:45:42.612522+0000","last_peered":"2024-09-13T14:45:42.612522+0000","last_clean":"2024-09-13T14:45:42.612522+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:45:42.612522+0000","last_undegraded":"2024-09-13T14:45:42.612522+0000","last_fullsized":"2024-09-13T14:45:42.612522+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.555310+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:45:34.555310+0000","last_peered":"2024-09-13T14:45:34.555310+0000","last_clean":"2024-09-13T14:45:34.555310+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:45:34.555310+0000","last_undegraded":"2024-09-13T14:45:34.555310+0000","last_fullsized":"2024-09-13T14:45:34.555310+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":319,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.528662+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:45:34.528662+0000","last_peered":"2024-09-13T14:45:34.528662+0000","last_clean":"2024-09-13T14:45:34.528662+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:45:34.528662+0000","last_undegraded":"2024-09-13T14:45:34.528662+0000","last_fullsized":"2024-09-13T14:45:34.528662+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":52,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.557188+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:45:34.557188+0000","last_peered":"2024-09-13T14:45:34.557188+0000","last_clean":"2024-09-13T14:45:34.557188+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:45:34.557188+0000","last_undegraded":"2024-09-13T14:45:34.557188+0000","last_fullsized":"2024-09-13T14:45:34.557188+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":285,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.529031+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:45:34.529031+0000","last_peered":"2024-09-13T14:45:34.529031+0000","last_clean":"2024-09-13T14:45:34.529031+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:45:34.529031+0000","last_undegraded":"2024-09-13T14:45:34.529031+0000","last_fullsized":"2024-09-13T14:45:34.529031+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":602,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.580428+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:45:42.580428+0000","last_peered":"2024-09-13T14:45:42.580428+0000","last_clean":"2024-09-13T14:45:42.580428+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:45:42.580428+0000","last_undegraded":"2024-09-13T14:45:42.580428+0000","last_fullsized":"2024-09-13T14:45:42.580428+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":588,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.581137+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:45:42.581137+0000","last_peered":"2024-09-13T14:45:42.581137+0000","last_clean":"2024-09-13T14:45:42.581137+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:45:42.581137+0000","last_undegraded":"2024-09-13T14:45:42.581137+0000","last_fullsized":"2024-09-13T14:45:42.581137+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-09-13T14:45:34.554316+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:45:34.554316+0000","last_peered":"2024-09-13T14:45:34.554316+0000","last_clean":"2024-09-13T14:45:34.554316+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:45:34.554316+0000","last_undegraded":"2024-09-13T14:45:34.554316+0000","last_fullsized":"2024-09-13T14:45:34.554316+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":593,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.611987+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:45:42.611987+0000","last_peered":"2024-09-13T14:45:42.611987+0000","last_clean":"2024-09-13T14:45:42.611987+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:45:42.611987+0000","last_undegraded":"2024-09-13T14:45:42.611987+0000","last_fullsized":"2024-09-13T14:45:42.611987+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3698589720,"num_objects":56437,"num_object_clones":0,"num_object_copies":169311,"num_objects_missing_on_primary":2506,"num_objects_missing":2506,"num_objects_degraded":36458,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":56437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126241,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24146,"num_bytes_recovered":1522008064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10250104832,"data_stored":10250092616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120741,"ondisk_log_size":120741,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397916,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2044444,"kb_used_data":1144276,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91700708,"statfs":{"total":95995035648,"available":93901524992,"internally_reserved":0,"allocated":1171738624,"data_stored":1169867827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594144,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2079496,"kb_used_data":1485440,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91665656,"statfs":{"total":95995035648,"available":93865631744,"internally_reserved":0,"allocated":1521090560,"data_stored":1519239517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665598,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615944,"kb_used_data":275072,"kb_used_omap":9,"kb_used_meta":340854,"kb_avail":93129208,"statfs":{"total":95995035648,"available":95364308992,"internally_reserved":0,"allocated":281673728,"data_stored":279819016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":349034696},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986599,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1729304,"kb_used_data":829452,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92015848,"statfs":{"total":95995035648,"available":94224228352,"internally_reserved":0,"allocated":849358848,"data_stored":847496149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,2,0,2,6,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697159,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490244,"kb_used_data":1736252,"kb_used_omap":4,"kb_used_meta":753979,"kb_avail":91254908,"statfs":{"total":95995035648,"available":93445025792,"internally_reserved":0,"allocated":1777922048,"data_stored":1776071489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":772074571},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860686,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270412,"kb_used_data":1649156,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":91474740,"statfs":{"total":95995035648,"available":93670133760,"internally_reserved":0,"allocated":1688735744,"data_stored":1686877531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020357,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1904256,"kb_used_data":1596536,"kb_used_omap":9,"kb_used_meta":307702,"kb_avail":91840896,"statfs":{"total":95995035648,"available":94045077504,"internally_reserved":0,"allocated":1634852864,"data_stored":1633005778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":315087685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187748,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1980992,"kb_used_data":1358008,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":91764160,"statfs":{"total":95995035648,"available":93966499840,"internally_reserved":0,"allocated":1390600192,"data_stored":1388753372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1378615296,"data_stored":1378615296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1624702976,"data_stored":1624702976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1683165184,"data_stored":1683161112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1769013248,"data_stored":1769013248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":843251712,"data_stored":843251712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":275386368,"data_stored":275382296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1516240896,"data_stored":1516240896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1159729152,"data_stored":1159725080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:50.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:50.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:50.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:50.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:50.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:50.719 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:50 smithi189 ceph-mon[90992]: pgmap v1104: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 36458/169395 objects degraded (21.522%); 3460/169395 objects misplaced (2.043%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:50.719 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:50 smithi189 ceph-mon[90992]: from='client.46316 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:50.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:50.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:50.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:50.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:50 smithi017 ceph-mon[103001]: pgmap v1104: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 36458/169395 objects degraded (21.522%); 3460/169395 objects misplaced (2.043%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:50.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:50 smithi017 ceph-mon[103001]: from='client.46316 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:50.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:50 smithi017 ceph-mon[107556]: pgmap v1104: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 36458/169395 objects degraded (21.522%); 3460/169395 objects misplaced (2.043%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:50.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:50 smithi017 ceph-mon[107556]: from='client.46316 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:50.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:50.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:50.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:50.984 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:50.985 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:51.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:51.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:51.141 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:51.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:51.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:51.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:51.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:51.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:51.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:51.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:51.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:51.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:51.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:51.778 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:51.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:51.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:45:51.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:51.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:52.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:52.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:52.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:52.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:52.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:52.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:52.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:52.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:52.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:52.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:52.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:52.577 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:45:52.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:52.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:52 smithi017 ceph-mon[103001]: pgmap v1105: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 36415/168027 objects degraded (21.672%); 3415/168027 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:52 smithi017 ceph-mon[107556]: pgmap v1105: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 36415/168027 objects degraded (21.672%); 3415/168027 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:52.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:52.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:52.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:52.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:52.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:52 smithi189 ceph-mon[90992]: pgmap v1105: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 36415/168027 objects degraded (21.672%); 3415/168027 objects misplaced (2.032%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:45:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:53.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:53.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:53.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.253 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:53.260 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.406 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:53.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:53.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:53.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:53.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[107556]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:45:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:53 smithi017 ceph-mon[103001]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:45:53.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:53.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:53.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:53.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:53 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4169652791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:53 smithi189 ceph-mon[90992]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:45:54.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:54.027 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:54.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:54.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:54.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:54.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:54.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:54.294 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:54.384 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:54.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:54.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:54.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:54.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:54.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:54.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:54.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:54 smithi017 ceph-mon[103001]: pgmap v1107: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 36301/167388 objects degraded (21.687%); 3415/167388 objects misplaced (2.040%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:54.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36301/167388 objects degraded (21.687%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:54.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:54 smithi017 ceph-mon[107556]: pgmap v1107: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 36301/167388 objects degraded (21.687%); 3415/167388 objects misplaced (2.040%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:54.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36301/167388 objects degraded (21.687%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:54.783 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:54.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:54.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:54 smithi189 ceph-mon[90992]: pgmap v1107: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 36301/167388 objects degraded (21.687%); 3415/167388 objects misplaced (2.040%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36301/167388 objects degraded (21.687%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:55.016 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:55.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:55.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:55.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:55.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:55.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:55.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:55.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:55.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:55.541 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:55.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:55.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:45:55.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:45:55.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:45:55.543 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:55.543 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:55.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:55.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:55.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:55.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:55.807 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:55.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:45:55.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2907805946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:55.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2907805946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:55.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:55.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2907805946' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:56.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:56.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:56.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:56.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:56.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:56.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:56.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:56.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:56.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:56.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:56.554 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:56.656 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:56.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:56 smithi017 ceph-mon[103001]: pgmap v1108: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36252/166284 objects degraded (21.801%); 3368/166284 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:56 smithi017 ceph-mon[107556]: pgmap v1108: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36252/166284 objects degraded (21.801%); 3368/166284 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:56.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:56.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:56.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:56.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:56.928 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:56.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:56 smithi189 ceph-mon[90992]: pgmap v1108: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36252/166284 objects degraded (21.801%); 3368/166284 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:57.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:57.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:57.155 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:57.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:57.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:57.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:57.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:57.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:57.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:57.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:57.608 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:45:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:45:57] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T14:45:57.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:57.680 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:57.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:57.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:57.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:57.847 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:57.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:57.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:58.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:58.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:58.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:58.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:58.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:58.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:58.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:58.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[103001]: pgmap v1109: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/165597 objects degraded (21.834%); 3368/165597 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:58 smithi017 ceph-mon[107556]: pgmap v1109: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/165597 objects degraded (21.834%); 3368/165597 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:45:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:58 smithi189 ceph-mon[90992]: pgmap v1109: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/165597 objects degraded (21.834%); 3368/165597 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:45:58.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:58.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:58.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:58.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:58.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:58.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:58.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:59.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:59.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:59.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:59.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:59.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:59.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:59.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:59.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:45:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36156/165597 objects degraded (21.834%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:45:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36156/165597 objects degraded (21.834%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:59.487 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:45:59.487 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:45:59.487 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:45:59.487 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:45:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:45:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36156/165597 objects degraded (21.834%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:45:59.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:59.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:59.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:59.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:59.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:45:59.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:59.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:59.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:59.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:59.943 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:45:59.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:00.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:00.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:00.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:00.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:00.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:00.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:00.317 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:00 smithi189 ceph-mon[90992]: pgmap v1110: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/164700 objects degraded (21.953%); 3368/164700 objects misplaced (2.045%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:00.407 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:00.419 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:00.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:00.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:00 smithi017 ceph-mon[103001]: pgmap v1110: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/164700 objects degraded (21.953%); 3368/164700 objects misplaced (2.045%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:00 smithi017 ceph-mon[107556]: pgmap v1110: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 36156/164700 objects degraded (21.953%); 3368/164700 objects misplaced (2.045%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:00.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:00.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:00.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:00.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:00.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:00.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:00.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:00.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:01.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:01.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:46:01.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:01.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:01.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:01.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:01.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:01.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:01.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:01.846 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:01.847 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:46:01.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:02.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:02.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:02.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:02.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:02.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:02.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:02.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:02.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:02.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:02.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:02 smithi017 ceph-mon[103001]: pgmap v1111: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36109/163311 objects degraded (22.111%); 3321/163311 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:02.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:02 smithi017 ceph-mon[107556]: pgmap v1111: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36109/163311 objects degraded (22.111%); 3321/163311 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:02.577 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:02.604 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1111,"stamp":"2024-09-13T14:46:01.233163+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3570371254,"num_objects":54437,"num_object_clones":0,"num_object_copies":163311,"num_objects_missing_on_primary":2367,"num_objects_missing":2367,"num_objects_degraded":36109,"num_objects_misplaced":3321,"num_objects_unfound":0,"num_objects_dirty":54437,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":129216,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24447,"num_bytes_recovered":1538164893,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123601,"ondisk_log_size":123601,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14806608,"kb_used_data":9774032,"kb_used_omap":44,"kb_used_meta":5032467,"kb_avail":735154608,"statfs":{"total":767960285184,"available":752798318592,"internally_reserved":0,"allocated":10008608768,"data_stored":9993704063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5153246455},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3,0,0,0,5,3,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-103022592,"num_objects":-1572,"num_object_clones":0,"num_object_copies":-4716,"num_objects_missing_on_primary":-96,"num_objects_missing":-96,"num_objects_degraded":-306,"num_objects_misplaced":-94,"num_objects_unfound":0,"num_objects_dirty":-1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1572,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":199,"num_bytes_recovered":13041664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002788"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":594,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579745+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:45:52.579745+0000","last_peered":"2024-09-13T14:45:52.579745+0000","last_clean":"2024-09-13T14:45:52.579745+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:45:52.579745+0000","last_undegraded":"2024-09-13T14:45:52.579745+0000","last_fullsized":"2024-09-13T14:45:52.579745+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":353,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562853+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:45:42.562853+0000","last_peered":"2024-09-13T14:45:42.562853+0000","last_clean":"2024-09-13T14:45:42.562853+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:45:42.562853+0000","last_undegraded":"2024-09-13T14:45:42.562853+0000","last_fullsized":"2024-09-13T14:45:42.562853+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":282,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551432+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:45:42.551432+0000","last_peered":"2024-09-13T14:45:42.551432+0000","last_clean":"2024-09-13T14:45:42.551432+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:45:42.551432+0000","last_undegraded":"2024-09-13T14:45:42.551432+0000","last_fullsized":"2024-09-13T14:45:42.551432+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":579,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.607852+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:45:52.607852+0000","last_peered":"2024-09-13T14:45:52.607852+0000","last_clean":"2024-09-13T14:45:52.607852+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:45:52.607852+0000","last_undegraded":"2024-09-13T14:45:52.607852+0000","last_fullsized":"2024-09-13T14:45:52.607852+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":580,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608435+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:45:52.608435+0000","last_peered":"2024-09-13T14:45:52.608435+0000","last_clean":"2024-09-13T14:45:52.608435+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:45:52.608435+0000","last_undegraded":"2024-09-13T14:45:52.608435+0000","last_fullsized":"2024-09-13T14:45:52.608435+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":322,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562757+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:45:42.562757+0000","last_peered":"2024-09-13T14:45:42.562757+0000","last_clean":"2024-09-13T14:45:42.562757+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:45:42.562757+0000","last_undegraded":"2024-09-13T14:45:42.562757+0000","last_fullsized":"2024-09-13T14:45:42.562757+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":594,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579024+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:45:52.579024+0000","last_peered":"2024-09-13T14:45:52.579024+0000","last_clean":"2024-09-13T14:45:52.579024+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:45:52.579024+0000","last_undegraded":"2024-09-13T14:45:52.579024+0000","last_fullsized":"2024-09-13T14:45:52.579024+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":344,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548446+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:45:42.548446+0000","last_peered":"2024-09-13T14:45:42.548446+0000","last_clean":"2024-09-13T14:45:42.548446+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:45:42.548446+0000","last_undegraded":"2024-09-13T14:45:42.548446+0000","last_fullsized":"2024-09-13T14:45:42.548446+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"355'7956","reported_seq":15000,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:56.901107+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:45:56.901107+0000","last_peered":"2024-09-13T14:45:56.901107+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:45:56.901107+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7956,"log_dups_size":0,"ondisk_log_size":7956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218497024,"num_objects":3334,"num_object_clones":0,"num_object_copies":10002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7956,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2311,"num_bytes_recovered":151453696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2350}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":348,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.563420+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:45:42.563420+0000","last_peered":"2024-09-13T14:45:42.563420+0000","last_clean":"2024-09-13T14:45:42.563420+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:45:42.563420+0000","last_undegraded":"2024-09-13T14:45:42.563420+0000","last_fullsized":"2024-09-13T14:45:42.563420+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":267,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551617+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:45:42.551617+0000","last_peered":"2024-09-13T14:45:42.551617+0000","last_clean":"2024-09-13T14:45:42.551617+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:45:42.551617+0000","last_undegraded":"2024-09-13T14:45:42.551617+0000","last_fullsized":"2024-09-13T14:45:42.551617+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"355'8018","reported_seq":17570,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:57.708631+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:45:57.708631+0000","last_peered":"2024-09-13T14:45:57.708631+0000","last_clean":"2024-09-13T14:45:57.708631+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:45:57.708631+0000","last_undegraded":"2024-09-13T14:45:57.708631+0000","last_fullsized":"2024-09-13T14:45:57.708631+0000","mapping_epoch":326,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8018,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":366,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.563228+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:45:42.563228+0000","last_peered":"2024-09-13T14:45:42.563228+0000","last_clean":"2024-09-13T14:45:42.563228+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:45:42.563228+0000","last_undegraded":"2024-09-13T14:45:42.563228+0000","last_fullsized":"2024-09-13T14:45:42.563228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569502+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:45:42.569502+0000","last_peered":"2024-09-13T14:45:42.569502+0000","last_clean":"2024-09-13T14:45:42.569502+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:45:42.569502+0000","last_undegraded":"2024-09-13T14:45:42.569502+0000","last_fullsized":"2024-09-13T14:45:42.569502+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"355'8313","reported_seq":8430,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.962681+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:45:58.962681+0000","last_peered":"2024-09-13T14:45:58.962681+0000","last_clean":"2024-09-13T14:45:58.962681+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:45:58.962681+0000","last_undegraded":"2024-09-13T14:45:58.962681+0000","last_fullsized":"2024-09-13T14:45:58.962681+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8313,"log_dups_size":0,"ondisk_log_size":8313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228392960,"num_objects":3485,"num_object_clones":0,"num_object_copies":10455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8313,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":320,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.566769+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:45:42.566769+0000","last_peered":"2024-09-13T14:45:42.566769+0000","last_clean":"2024-09-13T14:45:42.566769+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:45:42.566769+0000","last_undegraded":"2024-09-13T14:45:42.566769+0000","last_fullsized":"2024-09-13T14:45:42.566769+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":335,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548883+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:45:42.548883+0000","last_peered":"2024-09-13T14:45:42.548883+0000","last_clean":"2024-09-13T14:45:42.548883+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:45:42.548883+0000","last_undegraded":"2024-09-13T14:45:42.548883+0000","last_fullsized":"2024-09-13T14:45:42.548883+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"355'8130","reported_seq":8247,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:59.667319+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:45:59.667319+0000","last_peered":"2024-09-13T14:45:59.667319+0000","last_clean":"2024-09-13T14:45:59.667319+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:45:59.667319+0000","last_undegraded":"2024-09-13T14:45:59.667319+0000","last_fullsized":"2024-09-13T14:45:59.667319+0000","mapping_epoch":303,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223870976,"num_objects":3416,"num_object_clones":0,"num_object_copies":10248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8130,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":283,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551886+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:45:42.551886+0000","last_peered":"2024-09-13T14:45:42.551886+0000","last_clean":"2024-09-13T14:45:42.551886+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:45:42.551886+0000","last_undegraded":"2024-09-13T14:45:42.551886+0000","last_fullsized":"2024-09-13T14:45:42.551886+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":605,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608118+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:45:52.608118+0000","last_peered":"2024-09-13T14:45:52.608118+0000","last_clean":"2024-09-13T14:45:52.608118+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:45:52.608118+0000","last_undegraded":"2024-09-13T14:45:52.608118+0000","last_fullsized":"2024-09-13T14:45:52.608118+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"355'8064","reported_seq":8179,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:00.627319+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:46:00.627319+0000","last_peered":"2024-09-13T14:46:00.627319+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:46:00.627319+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222691328,"num_objects":3398,"num_object_clones":0,"num_object_copies":10194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8064,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2235}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":466,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562428+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:45:42.562428+0000","last_peered":"2024-09-13T14:45:42.562428+0000","last_clean":"2024-09-13T14:45:42.562428+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:45:42.562428+0000","last_undegraded":"2024-09-13T14:45:42.562428+0000","last_fullsized":"2024-09-13T14:45:42.562428+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":603,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.578802+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:45:52.578802+0000","last_peered":"2024-09-13T14:45:52.578802+0000","last_clean":"2024-09-13T14:45:52.578802+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:45:52.578802+0000","last_undegraded":"2024-09-13T14:45:52.578802+0000","last_fullsized":"2024-09-13T14:45:52.578802+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569833+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:45:42.569833+0000","last_peered":"2024-09-13T14:45:42.569833+0000","last_clean":"2024-09-13T14:45:42.569833+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:45:42.569833+0000","last_undegraded":"2024-09-13T14:45:42.569833+0000","last_fullsized":"2024-09-13T14:45:42.569833+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"355'7741","reported_seq":7859,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:59.632721+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:45:59.632721+0000","last_peered":"2024-09-13T14:45:59.632721+0000","last_clean":"2024-09-13T14:45:59.632721+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:45:59.632721+0000","last_undegraded":"2024-09-13T14:45:59.632721+0000","last_fullsized":"2024-09-13T14:45:59.632721+0000","mapping_epoch":303,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7741,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":170,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551665+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:45:42.551665+0000","last_peered":"2024-09-13T14:45:42.551665+0000","last_clean":"2024-09-13T14:45:42.551665+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:45:42.551665+0000","last_undegraded":"2024-09-13T14:45:42.551665+0000","last_fullsized":"2024-09-13T14:45:42.551665+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548717+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:45:42.548717+0000","last_peered":"2024-09-13T14:45:42.548717+0000","last_clean":"2024-09-13T14:45:42.548717+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:45:42.548717+0000","last_undegraded":"2024-09-13T14:45:42.548717+0000","last_fullsized":"2024-09-13T14:45:42.548717+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":68,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.572172+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:45:42.572172+0000","last_peered":"2024-09-13T14:45:42.572172+0000","last_clean":"2024-09-13T14:45:42.572172+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:45:42.572172+0000","last_undegraded":"2024-09-13T14:45:42.572172+0000","last_fullsized":"2024-09-13T14:45:42.572172+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"355'7988","reported_seq":14552,"reported_epoch":355,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:59.721218+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:45:59.721218+0000","last_peered":"2024-09-13T14:45:59.721218+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:45:59.721218+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224395264,"num_objects":3424,"num_object_clones":0,"num_object_copies":10272,"num_objects_missing_on_primary":2367,"num_objects_missing":2367,"num_objects_degraded":2742,"num_objects_misplaced":3321,"num_objects_unfound":0,"num_objects_dirty":3424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7988,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2282,"num_bytes_recovered":149553152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2367},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569283+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:45:42.569283+0000","last_peered":"2024-09-13T14:45:42.569283+0000","last_clean":"2024-09-13T14:45:42.569283+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:45:42.569283+0000","last_undegraded":"2024-09-13T14:45:42.569283+0000","last_fullsized":"2024-09-13T14:45:42.569283+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.570045+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:45:42.570045+0000","last_peered":"2024-09-13T14:45:42.570045+0000","last_clean":"2024-09-13T14:45:42.570045+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:45:42.570045+0000","last_undegraded":"2024-09-13T14:45:42.570045+0000","last_fullsized":"2024-09-13T14:45:42.570045+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"355'7846","reported_seq":7964,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:59.919167+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:45:59.919167+0000","last_peered":"2024-09-13T14:45:59.919167+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:45:59.919167+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217055232,"num_objects":3312,"num_object_clones":0,"num_object_copies":9936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7846,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2154}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":163,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551384+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:45:42.551384+0000","last_peered":"2024-09-13T14:45:42.551384+0000","last_clean":"2024-09-13T14:45:42.551384+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:45:42.551384+0000","last_undegraded":"2024-09-13T14:45:42.551384+0000","last_fullsized":"2024-09-13T14:45:42.551384+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":596,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.586479+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:45:52.586479+0000","last_peered":"2024-09-13T14:45:52.586479+0000","last_clean":"2024-09-13T14:45:52.586479+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:45:52.586479+0000","last_undegraded":"2024-09-13T14:45:52.586479+0000","last_fullsized":"2024-09-13T14:45:52.586479+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":558,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.590520+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:45:52.590520+0000","last_peered":"2024-09-13T14:45:52.590520+0000","last_clean":"2024-09-13T14:45:52.590520+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:45:52.590520+0000","last_undegraded":"2024-09-13T14:45:52.590520+0000","last_fullsized":"2024-09-13T14:45:52.590520+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"355'7859","reported_seq":7926,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:57.775953+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:45:57.775953+0000","last_peered":"2024-09-13T14:45:57.775953+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:45:57.775953+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7559,"log_dups_size":0,"ondisk_log_size":7559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220266496,"num_objects":3361,"num_object_clones":0,"num_object_copies":10083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7859,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2135}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":457,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562674+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:45:42.562674+0000","last_peered":"2024-09-13T14:45:42.562674+0000","last_clean":"2024-09-13T14:45:42.562674+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:45:42.562674+0000","last_undegraded":"2024-09-13T14:45:42.562674+0000","last_fullsized":"2024-09-13T14:45:42.562674+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":879,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551099+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:45:42.551099+0000","last_peered":"2024-09-13T14:45:42.551099+0000","last_clean":"2024-09-13T14:45:42.551099+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:45:42.551099+0000","last_undegraded":"2024-09-13T14:45:42.551099+0000","last_fullsized":"2024-09-13T14:45:42.551099+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":549,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.578889+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:45:52.578889+0000","last_peered":"2024-09-13T14:45:52.578889+0000","last_clean":"2024-09-13T14:45:52.578889+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:45:52.578889+0000","last_undegraded":"2024-09-13T14:45:52.578889+0000","last_fullsized":"2024-09-13T14:45:52.578889+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":321,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.549364+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:45:42.549364+0000","last_peered":"2024-09-13T14:45:42.549364+0000","last_clean":"2024-09-13T14:45:42.549364+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:45:42.549364+0000","last_undegraded":"2024-09-13T14:45:42.549364+0000","last_fullsized":"2024-09-13T14:45:42.549364+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"355'8151","reported_seq":8266,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:45:59.494359+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:45:59.494359+0000","last_peered":"2024-09-13T14:45:59.494359+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:45:59.494359+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7551,"log_dups_size":0,"ondisk_log_size":7551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222625792,"num_objects":3397,"num_object_clones":0,"num_object_copies":10191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8151,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2283}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":403,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.591831+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:45:52.591831+0000","last_peered":"2024-09-13T14:45:52.591831+0000","last_clean":"2024-09-13T14:45:52.591831+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:45:52.591831+0000","last_undegraded":"2024-09-13T14:45:52.591831+0000","last_fullsized":"2024-09-13T14:45:52.591831+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":745,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562591+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:45:42.562591+0000","last_peered":"2024-09-13T14:45:42.562591+0000","last_clean":"2024-09-13T14:45:42.562591+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:45:42.562591+0000","last_undegraded":"2024-09-13T14:45:42.562591+0000","last_fullsized":"2024-09-13T14:45:42.562591+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":598,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.610634+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:45:52.610634+0000","last_peered":"2024-09-13T14:45:52.610634+0000","last_clean":"2024-09-13T14:45:52.610634+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:45:52.610634+0000","last_undegraded":"2024-09-13T14:45:52.610634+0000","last_fullsized":"2024-09-13T14:45:52.610634+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":603,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608521+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:45:52.608521+0000","last_peered":"2024-09-13T14:45:52.608521+0000","last_clean":"2024-09-13T14:45:52.608521+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:45:52.608521+0000","last_undegraded":"2024-09-13T14:45:52.608521+0000","last_fullsized":"2024-09-13T14:45:52.608521+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"355'8142","reported_seq":17800,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:46:00.460483+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:46:00.460483+0000","last_peered":"2024-09-13T14:46:00.460483+0000","last_clean":"2024-09-13T14:46:00.460483+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:46:00.460483+0000","last_undegraded":"2024-09-13T14:46:00.460483+0000","last_fullsized":"2024-09-13T14:46:00.460483+0000","mapping_epoch":330,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230817792,"num_objects":3522,"num_object_clones":0,"num_object_copies":10566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8142,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":486,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.562591+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:45:42.562591+0000","last_peered":"2024-09-13T14:45:42.562591+0000","last_clean":"2024-09-13T14:45:42.562591+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:45:42.562591+0000","last_undegraded":"2024-09-13T14:45:42.562591+0000","last_fullsized":"2024-09-13T14:45:42.562591+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":567,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.578508+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:45:52.578508+0000","last_peered":"2024-09-13T14:45:52.578508+0000","last_clean":"2024-09-13T14:45:52.578508+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:45:52.578508+0000","last_undegraded":"2024-09-13T14:45:52.578508+0000","last_fullsized":"2024-09-13T14:45:52.578508+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":535,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589790+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:45:52.589790+0000","last_peered":"2024-09-13T14:45:52.589790+0000","last_clean":"2024-09-13T14:45:52.589790+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:45:52.589790+0000","last_undegraded":"2024-09-13T14:45:52.589790+0000","last_fullsized":"2024-09-13T14:45:52.589790+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":104,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551345+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:45:42.551345+0000","last_peered":"2024-09-13T14:45:42.551345+0000","last_clean":"2024-09-13T14:45:42.551345+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:45:42.551345+0000","last_undegraded":"2024-09-13T14:45:42.551345+0000","last_fullsized":"2024-09-13T14:45:42.551345+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"355'7858","reported_seq":7974,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:00.521158+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:46:00.521158+0000","last_peered":"2024-09-13T14:46:00.521158+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:46:00.521158+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220987392,"num_objects":3372,"num_object_clones":0,"num_object_copies":10116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7858,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2137}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":163,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.550981+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:45:42.550981+0000","last_peered":"2024-09-13T14:45:42.550981+0000","last_clean":"2024-09-13T14:45:42.550981+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:45:42.550981+0000","last_undegraded":"2024-09-13T14:45:42.550981+0000","last_fullsized":"2024-09-13T14:45:42.550981+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":970,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551456+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:45:42.551456+0000","last_peered":"2024-09-13T14:45:42.551456+0000","last_clean":"2024-09-13T14:45:42.551456+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:45:42.551456+0000","last_undegraded":"2024-09-13T14:45:42.551456+0000","last_fullsized":"2024-09-13T14:45:42.551456+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569666+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:45:42.569666+0000","last_peered":"2024-09-13T14:45:42.569666+0000","last_clean":"2024-09-13T14:45:42.569666+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:45:42.569666+0000","last_undegraded":"2024-09-13T14:45:42.569666+0000","last_fullsized":"2024-09-13T14:45:42.569666+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":556,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589750+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:45:52.589750+0000","last_peered":"2024-09-13T14:45:52.589750+0000","last_clean":"2024-09-13T14:45:52.589750+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:45:52.589750+0000","last_undegraded":"2024-09-13T14:45:52.589750+0000","last_fullsized":"2024-09-13T14:45:52.589750+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"355'7979","reported_seq":17376,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.751053+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:45:58.751053+0000","last_peered":"2024-09-13T14:45:58.751053+0000","last_clean":"2024-09-13T14:45:58.751053+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:45:58.751053+0000","last_undegraded":"2024-09-13T14:45:58.751053+0000","last_fullsized":"2024-09-13T14:45:58.751053+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7979,"log_dups_size":0,"ondisk_log_size":7979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221577216,"num_objects":3381,"num_object_clones":0,"num_object_copies":10143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7979,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":354,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.548802+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:45:42.548802+0000","last_peered":"2024-09-13T14:45:42.548802+0000","last_clean":"2024-09-13T14:45:42.548802+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:45:42.548802+0000","last_undegraded":"2024-09-13T14:45:42.548802+0000","last_fullsized":"2024-09-13T14:45:42.548802+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551323+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:45:42.551323+0000","last_peered":"2024-09-13T14:45:42.551323+0000","last_clean":"2024-09-13T14:45:42.551323+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:45:42.551323+0000","last_undegraded":"2024-09-13T14:45:42.551323+0000","last_fullsized":"2024-09-13T14:45:42.551323+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1135,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.590020+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:45:52.590020+0000","last_peered":"2024-09-13T14:45:52.590020+0000","last_clean":"2024-09-13T14:45:52.590020+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:45:52.590020+0000","last_undegraded":"2024-09-13T14:45:52.590020+0000","last_fullsized":"2024-09-13T14:45:52.590020+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":537,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589868+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:45:52.589868+0000","last_peered":"2024-09-13T14:45:52.589868+0000","last_clean":"2024-09-13T14:45:52.589868+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:45:52.589868+0000","last_undegraded":"2024-09-13T14:45:52.589868+0000","last_fullsized":"2024-09-13T14:45:52.589868+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"355'7853","reported_seq":7970,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:46:00.358656+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:46:00.358656+0000","last_peered":"2024-09-13T14:46:00.358656+0000","last_clean":"2024-09-13T14:46:00.358656+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:46:00.358656+0000","last_undegraded":"2024-09-13T14:46:00.358656+0000","last_fullsized":"2024-09-13T14:46:00.358656+0000","mapping_epoch":303,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219611136,"num_objects":3351,"num_object_clones":0,"num_object_copies":10053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7853,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":354,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.549139+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:45:42.549139+0000","last_peered":"2024-09-13T14:45:42.549139+0000","last_clean":"2024-09-13T14:45:42.549139+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:45:42.549139+0000","last_undegraded":"2024-09-13T14:45:42.549139+0000","last_fullsized":"2024-09-13T14:45:42.549139+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":546,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589921+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:45:52.589921+0000","last_peered":"2024-09-13T14:45:52.589921+0000","last_clean":"2024-09-13T14:45:52.589921+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:45:52.589921+0000","last_undegraded":"2024-09-13T14:45:52.589921+0000","last_fullsized":"2024-09-13T14:45:52.589921+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":749,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.566832+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:45:42.566832+0000","last_peered":"2024-09-13T14:45:42.566832+0000","last_clean":"2024-09-13T14:45:42.566832+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:45:42.566832+0000","last_undegraded":"2024-09-13T14:45:42.566832+0000","last_fullsized":"2024-09-13T14:45:42.566832+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":581,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608039+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:45:52.608039+0000","last_peered":"2024-09-13T14:45:52.608039+0000","last_clean":"2024-09-13T14:45:52.608039+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:45:52.608039+0000","last_undegraded":"2024-09-13T14:45:52.608039+0000","last_fullsized":"2024-09-13T14:45:52.608039+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"355'8240","reported_seq":8357,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.982795+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:45:58.982795+0000","last_peered":"2024-09-13T14:45:58.982795+0000","last_clean":"2024-09-13T14:45:58.982795+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:45:58.982795+0000","last_undegraded":"2024-09-13T14:45:58.982795+0000","last_fullsized":"2024-09-13T14:45:58.982795+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8240,"log_dups_size":0,"ondisk_log_size":8240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226099200,"num_objects":3450,"num_object_clones":0,"num_object_copies":10350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.571924+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:45:42.571924+0000","last_peered":"2024-09-13T14:45:42.571924+0000","last_clean":"2024-09-13T14:45:42.571924+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:45:42.571924+0000","last_undegraded":"2024-09-13T14:45:42.571924+0000","last_fullsized":"2024-09-13T14:45:42.571924+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":582,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579298+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:45:52.579298+0000","last_peered":"2024-09-13T14:45:52.579298+0000","last_clean":"2024-09-13T14:45:52.579298+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:45:52.579298+0000","last_undegraded":"2024-09-13T14:45:52.579298+0000","last_fullsized":"2024-09-13T14:45:52.579298+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":922,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.590159+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:45:52.590159+0000","last_peered":"2024-09-13T14:45:52.590159+0000","last_clean":"2024-09-13T14:45:52.590159+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:45:52.590159+0000","last_undegraded":"2024-09-13T14:45:52.590159+0000","last_fullsized":"2024-09-13T14:45:52.590159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":580,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.586322+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:45:52.586322+0000","last_peered":"2024-09-13T14:45:52.586322+0000","last_clean":"2024-09-13T14:45:52.586322+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:45:52.586322+0000","last_undegraded":"2024-09-13T14:45:52.586322+0000","last_fullsized":"2024-09-13T14:45:52.586322+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"355'8131","reported_seq":19497,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:57.753398+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:45:57.753398+0000","last_peered":"2024-09-13T14:45:57.753398+0000","last_clean":"2024-09-13T14:45:57.753398+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:45:57.753398+0000","last_undegraded":"2024-09-13T14:45:57.753398+0000","last_fullsized":"2024-09-13T14:45:57.753398+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221642776,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8131,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":433,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579596+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:45:52.579596+0000","last_peered":"2024-09-13T14:45:52.579596+0000","last_clean":"2024-09-13T14:45:52.579596+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:45:52.579596+0000","last_undegraded":"2024-09-13T14:45:52.579596+0000","last_fullsized":"2024-09-13T14:45:52.579596+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":617,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608378+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:45:52.608378+0000","last_peered":"2024-09-13T14:45:52.608378+0000","last_clean":"2024-09-13T14:45:52.608378+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:45:52.608378+0000","last_undegraded":"2024-09-13T14:45:52.608378+0000","last_fullsized":"2024-09-13T14:45:52.608378+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":773,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.549574+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:45:42.549574+0000","last_peered":"2024-09-13T14:45:42.549574+0000","last_clean":"2024-09-13T14:45:42.549574+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:45:42.549574+0000","last_undegraded":"2024-09-13T14:45:42.549574+0000","last_fullsized":"2024-09-13T14:45:42.549574+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569395+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:45:42.569395+0000","last_peered":"2024-09-13T14:45:42.569395+0000","last_clean":"2024-09-13T14:45:42.569395+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:45:42.569395+0000","last_undegraded":"2024-09-13T14:45:42.569395+0000","last_fullsized":"2024-09-13T14:45:42.569395+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569679+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:45:42.569679+0000","last_peered":"2024-09-13T14:45:42.569679+0000","last_clean":"2024-09-13T14:45:42.569679+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:45:42.569679+0000","last_undegraded":"2024-09-13T14:45:42.569679+0000","last_fullsized":"2024-09-13T14:45:42.569679+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":64,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569756+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:45:42.569756+0000","last_peered":"2024-09-13T14:45:42.569756+0000","last_clean":"2024-09-13T14:45:42.569756+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:45:42.569756+0000","last_undegraded":"2024-09-13T14:45:42.569756+0000","last_fullsized":"2024-09-13T14:45:42.569756+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":533,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589727+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:45:52.589727+0000","last_peered":"2024-09-13T14:45:52.589727+0000","last_clean":"2024-09-13T14:45:52.589727+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:45:52.589727+0000","last_undegraded":"2024-09-13T14:45:52.589727+0000","last_fullsized":"2024-09-13T14:45:52.589727+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":354,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.566682+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:45:42.566682+0000","last_peered":"2024-09-13T14:45:42.566682+0000","last_clean":"2024-09-13T14:45:42.566682+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:45:42.566682+0000","last_undegraded":"2024-09-13T14:45:42.566682+0000","last_fullsized":"2024-09-13T14:45:42.566682+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":320,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.550956+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:45:42.550956+0000","last_peered":"2024-09-13T14:45:42.550956+0000","last_clean":"2024-09-13T14:45:42.550956+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:45:42.550956+0000","last_undegraded":"2024-09-13T14:45:42.550956+0000","last_fullsized":"2024-09-13T14:45:42.550956+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":538,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.578334+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:45:52.578334+0000","last_peered":"2024-09-13T14:45:52.578334+0000","last_clean":"2024-09-13T14:45:52.578334+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:45:52.578334+0000","last_undegraded":"2024-09-13T14:45:52.578334+0000","last_fullsized":"2024-09-13T14:45:52.578334+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":590,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608028+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:45:52.608028+0000","last_peered":"2024-09-13T14:45:52.608028+0000","last_clean":"2024-09-13T14:45:52.608028+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:45:52.608028+0000","last_undegraded":"2024-09-13T14:45:52.608028+0000","last_fullsized":"2024-09-13T14:45:52.608028+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":594,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.586178+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:45:52.586178+0000","last_peered":"2024-09-13T14:45:52.586178+0000","last_clean":"2024-09-13T14:45:52.586178+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:45:52.586178+0000","last_undegraded":"2024-09-13T14:45:52.586178+0000","last_fullsized":"2024-09-13T14:45:52.586178+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":352,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.549486+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:45:42.549486+0000","last_peered":"2024-09-13T14:45:42.549486+0000","last_clean":"2024-09-13T14:45:42.549486+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:45:42.549486+0000","last_undegraded":"2024-09-13T14:45:42.549486+0000","last_fullsized":"2024-09-13T14:45:42.549486+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":612,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.586375+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:45:52.586375+0000","last_peered":"2024-09-13T14:45:52.586375+0000","last_clean":"2024-09-13T14:45:52.586375+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:45:52.586375+0000","last_undegraded":"2024-09-13T14:45:52.586375+0000","last_fullsized":"2024-09-13T14:45:52.586375+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":582,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.608466+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:45:52.608466+0000","last_peered":"2024-09-13T14:45:52.608466+0000","last_clean":"2024-09-13T14:45:52.608466+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:45:52.608466+0000","last_undegraded":"2024-09-13T14:45:52.608466+0000","last_fullsized":"2024-09-13T14:45:52.608466+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":558,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.589739+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:45:52.589739+0000","last_peered":"2024-09-13T14:45:52.589739+0000","last_clean":"2024-09-13T14:45:52.589739+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:45:52.589739+0000","last_undegraded":"2024-09-13T14:45:52.589739+0000","last_fullsized":"2024-09-13T14:45:52.589739+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":607,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579626+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:45:52.579626+0000","last_peered":"2024-09-13T14:45:52.579626+0000","last_clean":"2024-09-13T14:45:52.579626+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:45:52.579626+0000","last_undegraded":"2024-09-13T14:45:52.579626+0000","last_fullsized":"2024-09-13T14:45:52.579626+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":53,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.585363+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:45:42.585363+0000","last_peered":"2024-09-13T14:45:42.585363+0000","last_clean":"2024-09-13T14:45:42.585363+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:45:42.585363+0000","last_undegraded":"2024-09-13T14:45:42.585363+0000","last_fullsized":"2024-09-13T14:45:42.585363+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":504,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.590161+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:45:52.590161+0000","last_peered":"2024-09-13T14:45:52.590161+0000","last_clean":"2024-09-13T14:45:52.590161+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:45:52.590161+0000","last_undegraded":"2024-09-13T14:45:52.590161+0000","last_fullsized":"2024-09-13T14:45:52.590161+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":585,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.607932+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:45:52.607932+0000","last_peered":"2024-09-13T14:45:52.607932+0000","last_clean":"2024-09-13T14:45:52.607932+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:45:52.607932+0000","last_undegraded":"2024-09-13T14:45:52.607932+0000","last_fullsized":"2024-09-13T14:45:52.607932+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569612+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:45:42.569612+0000","last_peered":"2024-09-13T14:45:42.569612+0000","last_clean":"2024-09-13T14:45:42.569612+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:45:42.569612+0000","last_undegraded":"2024-09-13T14:45:42.569612+0000","last_fullsized":"2024-09-13T14:45:42.569612+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":320,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551955+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:45:42.551955+0000","last_peered":"2024-09-13T14:45:42.551955+0000","last_clean":"2024-09-13T14:45:42.551955+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:45:42.551955+0000","last_undegraded":"2024-09-13T14:45:42.551955+0000","last_fullsized":"2024-09-13T14:45:42.551955+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.585317+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:45:42.585317+0000","last_peered":"2024-09-13T14:45:42.585317+0000","last_clean":"2024-09-13T14:45:42.585317+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:45:42.585317+0000","last_undegraded":"2024-09-13T14:45:42.585317+0000","last_fullsized":"2024-09-13T14:45:42.585317+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":286,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.551508+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:45:42.551508+0000","last_peered":"2024-09-13T14:45:42.551508+0000","last_clean":"2024-09-13T14:45:42.551508+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:45:42.551508+0000","last_undegraded":"2024-09-13T14:45:42.551508+0000","last_fullsized":"2024-09-13T14:45:42.551508+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":604,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.578582+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:45:52.578582+0000","last_peered":"2024-09-13T14:45:52.578582+0000","last_clean":"2024-09-13T14:45:52.578582+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:45:52.578582+0000","last_undegraded":"2024-09-13T14:45:52.578582+0000","last_fullsized":"2024-09-13T14:45:52.578582+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":591,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.579395+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:45:52.579395+0000","last_peered":"2024-09-13T14:45:52.579395+0000","last_clean":"2024-09-13T14:45:52.579395+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:45:52.579395+0000","last_undegraded":"2024-09-13T14:45:52.579395+0000","last_fullsized":"2024-09-13T14:45:52.579395+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":354,"state":"active+clean","last_fresh":"2024-09-13T14:45:42.569133+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:45:42.569133+0000","last_peered":"2024-09-13T14:45:42.569133+0000","last_clean":"2024-09-13T14:45:42.569133+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:45:42.569133+0000","last_undegraded":"2024-09-13T14:45:42.569133+0000","last_fullsized":"2024-09-13T14:45:42.569133+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":595,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-09-13T14:45:52.607988+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:45:52.607988+0000","last_peered":"2024-09-13T14:45:52.607988+0000","last_clean":"2024-09-13T14:45:52.607988+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:45:52.607988+0000","last_undegraded":"2024-09-13T14:45:52.607988+0000","last_fullsized":"2024-09-13T14:45:52.607988+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3565682712,"num_objects":54409,"num_object_clones":0,"num_object_copies":163227,"num_objects_missing_on_primary":2367,"num_objects_missing":2367,"num_objects_degraded":36109,"num_objects_misplaced":3321,"num_objects_unfound":0,"num_objects_dirty":54409,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128269,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24390,"num_bytes_recovered":1537998848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9944444928,"data_stored":9944432712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121469,"ondisk_log_size":121469,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397918,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2017324,"kb_used_data":1112996,"kb_used_omap":1,"kb_used_meta":904318,"kb_avail":91727828,"statfs":{"total":95995035648,"available":93929295872,"internally_reserved":0,"allocated":1139707904,"data_stored":1137829260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926021976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594146,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2051544,"kb_used_data":1453328,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":91693608,"statfs":{"total":95995035648,"available":93894254592,"internally_reserved":0,"allocated":1488207872,"data_stored":1486348982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665600,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597144,"kb_used_data":268752,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93148008,"statfs":{"total":95995035648,"available":95383560192,"internally_reserved":0,"allocated":275202048,"data_stored":273339489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986602,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1701604,"kb_used_data":797596,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":92043548,"statfs":{"total":95995035648,"available":94252593152,"internally_reserved":0,"allocated":816738304,"data_stored":814867758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3,0,0,0,5,3,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697162,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2435604,"kb_used_data":1677452,"kb_used_omap":4,"kb_used_meta":758139,"kb_avail":91309548,"statfs":{"total":95995035648,"available":93500977152,"internally_reserved":0,"allocated":1717710848,"data_stored":1715852442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":776334411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860688,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231900,"kb_used_data":1610644,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":91513252,"statfs":{"total":95995035648,"available":93709570048,"internally_reserved":0,"allocated":1649299456,"data_stored":1647433396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020360,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827664,"kb_used_data":1532424,"kb_used_omap":9,"kb_used_meta":295222,"kb_avail":91917488,"statfs":{"total":95995035648,"available":94123507712,"internally_reserved":0,"allocated":1569202176,"data_stored":1567347243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":302308165},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187750,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1943824,"kb_used_data":1320840,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":91801328,"statfs":{"total":95995035648,"available":94004559872,"internally_reserved":0,"allocated":1352540160,"data_stored":1350685493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1340538880,"data_stored":1340538880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1557266432,"data_stored":1557266432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1644302336,"data_stored":1644298264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1709637632,"data_stored":1709637632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":812056576,"data_stored":812056576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":269488128,"data_stored":269484056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1483603968,"data_stored":1483603968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1127550976,"data_stored":1127546904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:02.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:46:02.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:02.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:02 smithi189 ceph-mon[90992]: pgmap v1111: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 36109/163311 objects degraded (22.111%); 3321/163311 objects misplaced (2.034%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:02.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:02.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:02.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:02.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:02.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:02.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:03.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:03.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:03.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:03.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:03.293 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:46:03.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:03.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:03.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:03.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:03.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:03.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:03.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:03.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[103001]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3449364707' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[107556]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3449364707' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:03 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:03 smithi189 ceph-mon[90992]: from='client.36465 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3449364707' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:03 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:03.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:03.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:03.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:03.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:04.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:04.311 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:04.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:04.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:04.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[107556]: pgmap v1112: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 36029/162744 objects degraded (22.138%); 3321/162744 objects misplaced (2.041%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[107556]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 36029/162744 objects degraded (22.138%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[103001]: pgmap v1112: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 36029/162744 objects degraded (22.138%); 3321/162744 objects misplaced (2.041%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[103001]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:46:04.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 36029/162744 objects degraded (22.138%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:04.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:04.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:04.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:04 smithi189 ceph-mon[90992]: pgmap v1112: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 36029/162744 objects degraded (22.138%); 3321/162744 objects misplaced (2.041%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:04 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:04 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:04 smithi189 ceph-mon[90992]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:46:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 36029/162744 objects degraded (22.138%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:04.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:04.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:04.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:04.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:05.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:05.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:05.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:05.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:05.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:05.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:05.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:05.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:05.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:05.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:05.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:05.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:05.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:05.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:05.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:05.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:06.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:46:06.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:06.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:06.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:06.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:06.300 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:06.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:06 smithi017 ceph-mon[107556]: pgmap v1114: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 35985/161628 objects degraded (22.264%); 3273/161628 objects misplaced (2.025%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:46:06.310 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:06 smithi017 ceph-mon[103001]: pgmap v1114: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 35985/161628 objects degraded (22.264%); 3273/161628 objects misplaced (2.025%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:46:06.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:06.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:06.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:06.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:06.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:06.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:06.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:06.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:06 smithi189 ceph-mon[90992]: pgmap v1114: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 35985/161628 objects degraded (22.264%); 3273/161628 objects misplaced (2.025%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T14:46:06.801 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:46:06.802 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:06.806 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:46:06.806 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:06.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:06.828 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:46:06.829 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:06.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:06.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:06.849 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:46:06.849 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:06.866 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:46:06.866 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:06.883 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:46:06.883 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:06.899 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:46:06.899 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:06.917 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:46:06.917 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:06.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:06.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:07.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:07.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:07.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:07.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:07.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:07.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:07.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:07.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.466 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:07.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:07.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3781889837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:07.584 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:07] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T14:46:07.584 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3781889837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:07.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:07.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:07.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:07.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3781889837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:07.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:07.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:07.948 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:08.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:08.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:08.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:08.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:08 smithi017 ceph-mon[103001]: pgmap v1115: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 35905/161001 objects degraded (22.301%); 3273/161001 objects misplaced (2.033%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:08.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:08 smithi017 ceph-mon[107556]: pgmap v1115: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 35905/161001 objects degraded (22.301%); 3273/161001 objects misplaced (2.033%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:08.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:08.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:08.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:08.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:08.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:08.696 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:08.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:08 smithi189 ceph-mon[90992]: pgmap v1115: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 35905/161001 objects degraded (22.301%); 3273/161001 objects misplaced (2.033%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:08.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:08.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:08.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:08.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:09.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:09.087 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:09.097 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:09.097 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:09.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:09.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:09.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:09.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:09.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:09.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:09.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35905/161001 objects degraded (22.301%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:09.415 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35905/161001 objects degraded (22.301%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:09.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:09.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:09.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:09.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:09.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:09.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35905/161001 objects degraded (22.301%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:09.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:09.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:09.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:09.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:09.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:09.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:46:10.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:10.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:10.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:10.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:10.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:10.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:10.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:10.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:10.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:10 smithi017 ceph-mon[103001]: pgmap v1116: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35905/160275 objects degraded (22.402%); 3273/160275 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:10.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:10 smithi017 ceph-mon[107556]: pgmap v1116: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35905/160275 objects degraded (22.402%); 3273/160275 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:10 smithi189 ceph-mon[90992]: pgmap v1116: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35905/160275 objects degraded (22.402%); 3273/160275 objects misplaced (2.042%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:10.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:10.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:10.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:10.730 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:10.731 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35905/160275 objects degraded (22.402%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1568,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":356,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":53425,"data_bytes":3504048822,"bytes_used":14946684928,"bytes_avail":753013600256,"bytes_total":767960285184,"degraded_objects":35905,"degraded_total":160275,"degraded_ratio":0.22402121353922944,"misplaced_objects":3273,"misplaced_total":160275,"misplaced_ratio":0.020421151146467011,"recovering_objects_per_sec":17,"recovering_bytes_per_sec":1146556,"recovering_keys_per_sec":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":147},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (7m)\n [==========================..] (remaining: 31s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:46:10.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:10.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:10.836 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:10.857 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:10.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:10.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:10.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:11.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:11.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:11.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:11.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:11.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:11.354 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:11.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:11.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:11.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:11.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:11.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:11.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4221143935' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4221143935' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:11.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:11.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:11.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4221143935' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:11.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:11.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:11.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:11.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:12.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:12.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:12.211 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:12.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:12.334 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:12.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:12.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:12.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:12.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:12.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[103001]: pgmap v1117: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35855/158889 objects degraded (22.566%); 3224/158889 objects misplaced (2.029%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[107556]: pgmap v1117: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35855/158889 objects degraded (22.566%); 3224/158889 objects misplaced (2.029%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:12.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:12.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:12.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:12 smithi189 ceph-mon[90992]: pgmap v1117: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 35855/158889 objects degraded (22.566%); 3224/158889 objects misplaced (2.029%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:12.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:12.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:12.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:12.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:13.184 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:13.205 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:46:13.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:13.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:13.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:13.316 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:46:13.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:13.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:13 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:13 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:13.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:13.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:13.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:13.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:13.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:13.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:13 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:13.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:13.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:13.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:13.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:14.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:14.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:14.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:14.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:14.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:14.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[103001]: pgmap v1118: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35737/158208 objects degraded (22.589%); 3224/158208 objects misplaced (2.038%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[103001]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:46:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35737/158208 objects degraded (22.589%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[107556]: pgmap v1118: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35737/158208 objects degraded (22.589%); 3224/158208 objects misplaced (2.038%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[107556]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:46:14.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35737/158208 objects degraded (22.589%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:14.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:14.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:14.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:14.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:14.687 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:14.706 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:14.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:14.731 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:14 smithi189 ceph-mon[90992]: pgmap v1118: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35737/158208 objects degraded (22.589%); 3224/158208 objects misplaced (2.038%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:14 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3609593403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:14 smithi189 ceph-mon[90992]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:46:14.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35737/158208 objects degraded (22.589%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:14.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:14.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:14.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:14.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:15.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:15.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:15.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:15.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:15.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:15.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:15.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:15.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:15.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:15.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:15.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:15.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:15.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:15.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:15.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:15.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:15.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:16.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:16.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:16.209 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:16.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:46:16.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:16.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:16.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:16.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:16.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:16.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:16.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:16.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:16 smithi017 ceph-mon[103001]: pgmap v1120: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 35688/156981 objects degraded (22.734%); 3176/156981 objects misplaced (2.023%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:16.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:16 smithi017 ceph-mon[107556]: pgmap v1120: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 35688/156981 objects degraded (22.734%); 3176/156981 objects misplaced (2.023%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:16.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:16.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:16.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:16 smithi189 ceph-mon[90992]: pgmap v1120: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 35688/156981 objects degraded (22.734%); 3176/156981 objects misplaced (2.023%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:16.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:16.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:16.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:16.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:16.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:17.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:17.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:17.209 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:17.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:17.332 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:17.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:17.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:17.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:17.586 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:17] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-13T14:46:17.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:17.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:17.719 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:17.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:17.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:17.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:17.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:17.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:17.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:18.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:18.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:18.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:18.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:18.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:18.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:18.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:18.474 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:18.474 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:18.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:18.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:18.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:18.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:18 smithi017 ceph-mon[103001]: pgmap v1121: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 35606/156249 objects degraded (22.788%); 3176/156249 objects misplaced (2.033%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:18 smithi017 ceph-mon[107556]: pgmap v1121: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 35606/156249 objects degraded (22.788%); 3176/156249 objects misplaced (2.033%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:18.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:18.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:18 smithi189 ceph-mon[90992]: pgmap v1121: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 35606/156249 objects degraded (22.788%); 3176/156249 objects misplaced (2.033%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:18.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:18.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:18.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:18.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:19.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:19.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:19.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:19.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:19.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:19.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:19.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35606/156249 objects degraded (22.788%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:19.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35606/156249 objects degraded (22.788%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:19.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:19.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:19.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:19.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:19.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:19.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35606/156249 objects degraded (22.788%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:19.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:19.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:19.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:19.978 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:46:19.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:19.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:20.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:20.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:20.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:20.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:20.272 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:46:20.322 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:20 smithi189 ceph-mon[90992]: pgmap v1122: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 35606/155442 objects degraded (22.906%); 3176/155442 objects misplaced (2.043%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:20.322 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:20.345 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:20.346 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:46:20.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:20.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:20.416 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:46:20.416 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:20.433 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:46:20.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:20.450 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:46:20.450 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:20.466 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:46:20.466 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:20.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:20.481 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:46:20.482 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:20.498 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:46:20.499 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:20.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:20.514 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:46:20.514 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:20.531 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:46:20.532 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:20.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:20.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:20 smithi017 ceph-mon[103001]: pgmap v1122: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 35606/155442 objects degraded (22.906%); 3176/155442 objects misplaced (2.043%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:20 smithi017 ceph-mon[107556]: pgmap v1122: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 35606/155442 objects degraded (22.906%); 3176/155442 objects misplaced (2.043%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:20.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:20.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:20.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:20.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:20.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:21.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:21.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:21.091 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1122,"stamp":"2024-09-13T14:46:19.238252+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3398470326,"num_objects":51814,"num_object_clones":0,"num_object_copies":155442,"num_objects_missing_on_primary":2224,"num_objects_missing":2224,"num_objects_degraded":35606,"num_objects_misplaced":3176,"num_objects_unfound":0,"num_objects_dirty":51814,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":131839,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24772,"num_bytes_recovered":1559464093,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124324,"ondisk_log_size":124324,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14329304,"kb_used_data":9342480,"kb_used_omap":44,"kb_used_meta":4986707,"kb_avail":735631912,"statfs":{"total":767960285184,"available":753287077888,"internally_reserved":0,"allocated":9566699520,"data_stored":9551669263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5106388215},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-105578496,"num_objects":-1611,"num_object_clones":0,"num_object_copies":-4833,"num_objects_missing_on_primary":-99,"num_objects_missing":-99,"num_objects_degraded":-299,"num_objects_misplaced":-97,"num_objects_unfound":0,"num_objects_dirty":-1611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1611,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":197,"num_bytes_recovered":12910592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002774"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":598,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.327006+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:46:13.327006+0000","last_peered":"2024-09-13T14:46:13.327006+0000","last_clean":"2024-09-13T14:46:13.327006+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:46:13.327006+0000","last_undegraded":"2024-09-13T14:46:13.327006+0000","last_fullsized":"2024-09-13T14:46:13.327006+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":355,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381642+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:46:03.381642+0000","last_peered":"2024-09-13T14:46:03.381642+0000","last_clean":"2024-09-13T14:46:03.381642+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:46:03.381642+0000","last_undegraded":"2024-09-13T14:46:03.381642+0000","last_fullsized":"2024-09-13T14:46:03.381642+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":284,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293336+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:46:03.293336+0000","last_peered":"2024-09-13T14:46:03.293336+0000","last_clean":"2024-09-13T14:46:03.293336+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:46:03.293336+0000","last_undegraded":"2024-09-13T14:46:03.293336+0000","last_fullsized":"2024-09-13T14:46:03.293336+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":583,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325054+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:46:13.325054+0000","last_peered":"2024-09-13T14:46:13.325054+0000","last_clean":"2024-09-13T14:46:13.325054+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:46:13.325054+0000","last_undegraded":"2024-09-13T14:46:13.325054+0000","last_fullsized":"2024-09-13T14:46:13.325054+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":584,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.327087+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:46:13.327087+0000","last_peered":"2024-09-13T14:46:13.327087+0000","last_clean":"2024-09-13T14:46:13.327087+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:46:13.327087+0000","last_undegraded":"2024-09-13T14:46:13.327087+0000","last_fullsized":"2024-09-13T14:46:13.327087+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":324,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381942+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:46:03.381942+0000","last_peered":"2024-09-13T14:46:03.381942+0000","last_clean":"2024-09-13T14:46:03.381942+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:46:03.381942+0000","last_undegraded":"2024-09-13T14:46:03.381942+0000","last_fullsized":"2024-09-13T14:46:03.381942+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":598,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326693+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:46:13.326693+0000","last_peered":"2024-09-13T14:46:13.326693+0000","last_clean":"2024-09-13T14:46:13.326693+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:46:13.326693+0000","last_undegraded":"2024-09-13T14:46:13.326693+0000","last_fullsized":"2024-09-13T14:46:13.326693+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.291884+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:46:03.291884+0000","last_peered":"2024-09-13T14:46:03.291884+0000","last_clean":"2024-09-13T14:46:03.291884+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:46:03.291884+0000","last_undegraded":"2024-09-13T14:46:03.291884+0000","last_fullsized":"2024-09-13T14:46:03.291884+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"357'8136","reported_seq":15724,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:16.390958+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:46:16.390958+0000","last_peered":"2024-09-13T14:46:16.390958+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:46:16.390958+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8136,"log_dups_size":0,"ondisk_log_size":8136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206700544,"num_objects":3154,"num_object_clones":0,"num_object_copies":9462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5223,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8136,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2491,"num_bytes_recovered":163250176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2170}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":350,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381891+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:46:03.381891+0000","last_peered":"2024-09-13T14:46:03.381891+0000","last_clean":"2024-09-13T14:46:03.381891+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:46:03.381891+0000","last_undegraded":"2024-09-13T14:46:03.381891+0000","last_fullsized":"2024-09-13T14:46:03.381891+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293102+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:46:03.293102+0000","last_peered":"2024-09-13T14:46:03.293102+0000","last_clean":"2024-09-13T14:46:03.293102+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:46:03.293102+0000","last_undegraded":"2024-09-13T14:46:03.293102+0000","last_fullsized":"2024-09-13T14:46:03.293102+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"357'8249","reported_seq":17803,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:17.491797+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:46:17.491797+0000","last_peered":"2024-09-13T14:46:17.491797+0000","last_clean":"2024-09-13T14:46:17.491797+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:46:17.491797+0000","last_undegraded":"2024-09-13T14:46:17.491797+0000","last_fullsized":"2024-09-13T14:46:17.491797+0000","mapping_epoch":326,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212271104,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8249,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":368,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381825+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:46:03.381825+0000","last_peered":"2024-09-13T14:46:03.381825+0000","last_clean":"2024-09-13T14:46:03.381825+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:46:03.381825+0000","last_undegraded":"2024-09-13T14:46:03.381825+0000","last_fullsized":"2024-09-13T14:46:03.381825+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292102+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:46:03.292102+0000","last_peered":"2024-09-13T14:46:03.292102+0000","last_clean":"2024-09-13T14:46:03.292102+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:46:03.292102+0000","last_undegraded":"2024-09-13T14:46:03.292102+0000","last_fullsized":"2024-09-13T14:46:03.292102+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"357'8492","reported_seq":8614,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:18.608384+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:46:18.608384+0000","last_peered":"2024-09-13T14:46:18.608384+0000","last_clean":"2024-09-13T14:46:18.608384+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:46:18.608384+0000","last_undegraded":"2024-09-13T14:46:18.608384+0000","last_fullsized":"2024-09-13T14:46:18.608384+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8492,"log_dups_size":0,"ondisk_log_size":8492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8492,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":322,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381958+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:46:03.381958+0000","last_peered":"2024-09-13T14:46:03.381958+0000","last_clean":"2024-09-13T14:46:03.381958+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:46:03.381958+0000","last_undegraded":"2024-09-13T14:46:03.381958+0000","last_fullsized":"2024-09-13T14:46:03.381958+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":337,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292456+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:46:03.292456+0000","last_peered":"2024-09-13T14:46:03.292456+0000","last_clean":"2024-09-13T14:46:03.292456+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:46:03.292456+0000","last_undegraded":"2024-09-13T14:46:03.292456+0000","last_fullsized":"2024-09-13T14:46:03.292456+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"357'8291","reported_seq":8413,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:14.575092+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:46:14.575092+0000","last_peered":"2024-09-13T14:46:14.575092+0000","last_clean":"2024-09-13T14:46:14.575092+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:46:14.575092+0000","last_undegraded":"2024-09-13T14:46:14.575092+0000","last_fullsized":"2024-09-13T14:46:14.575092+0000","mapping_epoch":303,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7591,"log_dups_size":0,"ondisk_log_size":7591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213319680,"num_objects":3255,"num_object_clones":0,"num_object_copies":9765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8291,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":285,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293452+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:46:03.293452+0000","last_peered":"2024-09-13T14:46:03.293452+0000","last_clean":"2024-09-13T14:46:03.293452+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:46:03.293452+0000","last_undegraded":"2024-09-13T14:46:03.293452+0000","last_fullsized":"2024-09-13T14:46:03.293452+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":609,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325156+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:46:13.325156+0000","last_peered":"2024-09-13T14:46:13.325156+0000","last_clean":"2024-09-13T14:46:13.325156+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:46:13.325156+0000","last_undegraded":"2024-09-13T14:46:13.325156+0000","last_fullsized":"2024-09-13T14:46:13.325156+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"357'8238","reported_seq":8357,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:15.486676+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:46:15.486676+0000","last_peered":"2024-09-13T14:46:15.486676+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:46:15.486676+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211288064,"num_objects":3224,"num_object_clones":0,"num_object_copies":9672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8238,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2409}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":468,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381450+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:46:03.381450+0000","last_peered":"2024-09-13T14:46:03.381450+0000","last_clean":"2024-09-13T14:46:03.381450+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:46:03.381450+0000","last_undegraded":"2024-09-13T14:46:03.381450+0000","last_fullsized":"2024-09-13T14:46:03.381450+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":607,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325961+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:46:13.325961+0000","last_peered":"2024-09-13T14:46:13.325961+0000","last_clean":"2024-09-13T14:46:13.325961+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:46:13.325961+0000","last_undegraded":"2024-09-13T14:46:13.325961+0000","last_fullsized":"2024-09-13T14:46:13.325961+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292357+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:46:03.292357+0000","last_peered":"2024-09-13T14:46:03.292357+0000","last_clean":"2024-09-13T14:46:03.292357+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:46:03.292357+0000","last_undegraded":"2024-09-13T14:46:03.292357+0000","last_fullsized":"2024-09-13T14:46:03.292357+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"357'7878","reported_seq":8000,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:14.852686+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:46:14.852686+0000","last_peered":"2024-09-13T14:46:14.852686+0000","last_clean":"2024-09-13T14:46:14.852686+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:46:14.852686+0000","last_undegraded":"2024-09-13T14:46:14.852686+0000","last_fullsized":"2024-09-13T14:46:14.852686+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210763776,"num_objects":3216,"num_object_clones":0,"num_object_copies":9648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7878,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":172,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293174+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:46:03.293174+0000","last_peered":"2024-09-13T14:46:03.293174+0000","last_clean":"2024-09-13T14:46:03.293174+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:46:03.293174+0000","last_undegraded":"2024-09-13T14:46:03.293174+0000","last_fullsized":"2024-09-13T14:46:03.293174+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":358,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292340+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:46:03.292340+0000","last_peered":"2024-09-13T14:46:03.292340+0000","last_clean":"2024-09-13T14:46:03.292340+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:46:03.292340+0000","last_undegraded":"2024-09-13T14:46:03.292340+0000","last_fullsized":"2024-09-13T14:46:03.292340+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":70,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292878+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:46:03.292878+0000","last_peered":"2024-09-13T14:46:03.292878+0000","last_clean":"2024-09-13T14:46:03.292878+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:46:03.292878+0000","last_undegraded":"2024-09-13T14:46:03.292878+0000","last_fullsized":"2024-09-13T14:46:03.292878+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"357'8133","reported_seq":15131,"reported_epoch":357,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:14.782888+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:46:14.782888+0000","last_peered":"2024-09-13T14:46:14.782888+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:46:14.782888+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214892544,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":2224,"num_objects_missing":2224,"num_objects_degraded":2599,"num_objects_misplaced":3176,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8133,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2427,"num_bytes_recovered":159055872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2224},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.291834+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:46:03.291834+0000","last_peered":"2024-09-13T14:46:03.291834+0000","last_clean":"2024-09-13T14:46:03.291834+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:46:03.291834+0000","last_undegraded":"2024-09-13T14:46:03.291834+0000","last_fullsized":"2024-09-13T14:46:03.291834+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292148+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:46:03.292148+0000","last_peered":"2024-09-13T14:46:03.292148+0000","last_clean":"2024-09-13T14:46:03.292148+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:46:03.292148+0000","last_undegraded":"2024-09-13T14:46:03.292148+0000","last_fullsized":"2024-09-13T14:46:03.292148+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"357'7975","reported_seq":8097,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:14.735162+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:46:14.735162+0000","last_peered":"2024-09-13T14:46:14.735162+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:46:14.735162+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208601088,"num_objects":3183,"num_object_clones":0,"num_object_copies":9549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7975,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2283}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":165,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292958+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:46:03.292958+0000","last_peered":"2024-09-13T14:46:03.292958+0000","last_clean":"2024-09-13T14:46:03.292958+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:46:03.292958+0000","last_undegraded":"2024-09-13T14:46:03.292958+0000","last_fullsized":"2024-09-13T14:46:03.292958+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":600,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.335048+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:46:13.335048+0000","last_peered":"2024-09-13T14:46:13.335048+0000","last_clean":"2024-09-13T14:46:13.335048+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:46:13.335048+0000","last_undegraded":"2024-09-13T14:46:13.335048+0000","last_fullsized":"2024-09-13T14:46:13.335048+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":562,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.339212+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:46:13.339212+0000","last_peered":"2024-09-13T14:46:13.339212+0000","last_clean":"2024-09-13T14:46:13.339212+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:46:13.339212+0000","last_undegraded":"2024-09-13T14:46:13.339212+0000","last_fullsized":"2024-09-13T14:46:13.339212+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"357'8031","reported_seq":8100,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:17.442777+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:46:17.442777+0000","last_peered":"2024-09-13T14:46:17.442777+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:46:17.442777+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208994304,"num_objects":3189,"num_object_clones":0,"num_object_copies":9567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8031,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2307}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":459,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381534+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:46:03.381534+0000","last_peered":"2024-09-13T14:46:03.381534+0000","last_clean":"2024-09-13T14:46:03.381534+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:46:03.381534+0000","last_undegraded":"2024-09-13T14:46:03.381534+0000","last_fullsized":"2024-09-13T14:46:03.381534+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":881,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292959+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:46:03.292959+0000","last_peered":"2024-09-13T14:46:03.292959+0000","last_clean":"2024-09-13T14:46:03.292959+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:46:03.292959+0000","last_undegraded":"2024-09-13T14:46:03.292959+0000","last_fullsized":"2024-09-13T14:46:03.292959+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":553,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326414+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:46:13.326414+0000","last_peered":"2024-09-13T14:46:13.326414+0000","last_clean":"2024-09-13T14:46:13.326414+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:46:13.326414+0000","last_undegraded":"2024-09-13T14:46:13.326414+0000","last_fullsized":"2024-09-13T14:46:13.326414+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":323,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293969+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:46:03.293969+0000","last_peered":"2024-09-13T14:46:03.293969+0000","last_clean":"2024-09-13T14:46:03.293969+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:46:03.293969+0000","last_undegraded":"2024-09-13T14:46:03.293969+0000","last_fullsized":"2024-09-13T14:46:03.293969+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"357'8301","reported_seq":8421,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:14.736097+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:46:14.736097+0000","last_peered":"2024-09-13T14:46:14.736097+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:46:14.736097+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212795392,"num_objects":3247,"num_object_clones":0,"num_object_copies":9741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8301,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2433}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":407,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.336969+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:46:13.336969+0000","last_peered":"2024-09-13T14:46:13.336969+0000","last_clean":"2024-09-13T14:46:13.336969+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:46:13.336969+0000","last_undegraded":"2024-09-13T14:46:13.336969+0000","last_fullsized":"2024-09-13T14:46:13.336969+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":747,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381503+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:46:03.381503+0000","last_peered":"2024-09-13T14:46:03.381503+0000","last_clean":"2024-09-13T14:46:03.381503+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:46:03.381503+0000","last_undegraded":"2024-09-13T14:46:03.381503+0000","last_fullsized":"2024-09-13T14:46:03.381503+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":602,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325214+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:46:13.325214+0000","last_peered":"2024-09-13T14:46:13.325214+0000","last_clean":"2024-09-13T14:46:13.325214+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:46:13.325214+0000","last_undegraded":"2024-09-13T14:46:13.325214+0000","last_fullsized":"2024-09-13T14:46:13.325214+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":607,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.327233+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:46:13.327233+0000","last_peered":"2024-09-13T14:46:13.327233+0000","last_clean":"2024-09-13T14:46:13.327233+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:46:13.327233+0000","last_undegraded":"2024-09-13T14:46:13.327233+0000","last_fullsized":"2024-09-13T14:46:13.327233+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"357'8284","reported_seq":17946,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:15.399999+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:46:15.399999+0000","last_peered":"2024-09-13T14:46:15.399999+0000","last_clean":"2024-09-13T14:46:15.399999+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:46:15.399999+0000","last_undegraded":"2024-09-13T14:46:15.399999+0000","last_fullsized":"2024-09-13T14:46:15.399999+0000","mapping_epoch":330,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221511680,"num_objects":3380,"num_object_clones":0,"num_object_copies":10140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8284,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":488,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381578+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:46:03.381578+0000","last_peered":"2024-09-13T14:46:03.381578+0000","last_clean":"2024-09-13T14:46:03.381578+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:46:03.381578+0000","last_undegraded":"2024-09-13T14:46:03.381578+0000","last_fullsized":"2024-09-13T14:46:03.381578+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":571,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326168+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:46:13.326168+0000","last_peered":"2024-09-13T14:46:13.326168+0000","last_clean":"2024-09-13T14:46:13.326168+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:46:13.326168+0000","last_undegraded":"2024-09-13T14:46:13.326168+0000","last_fullsized":"2024-09-13T14:46:13.326168+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":539,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.336868+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:46:13.336868+0000","last_peered":"2024-09-13T14:46:13.336868+0000","last_clean":"2024-09-13T14:46:13.336868+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:46:13.336868+0000","last_undegraded":"2024-09-13T14:46:13.336868+0000","last_fullsized":"2024-09-13T14:46:13.336868+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293512+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:46:03.293512+0000","last_peered":"2024-09-13T14:46:03.293512+0000","last_clean":"2024-09-13T14:46:03.293512+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:46:03.293512+0000","last_undegraded":"2024-09-13T14:46:03.293512+0000","last_fullsized":"2024-09-13T14:46:03.293512+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"357'7980","reported_seq":8101,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:15.496303+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:46:15.496303+0000","last_peered":"2024-09-13T14:46:15.496303+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:46:15.496303+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7980,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2259}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":165,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293617+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:46:03.293617+0000","last_peered":"2024-09-13T14:46:03.293617+0000","last_clean":"2024-09-13T14:46:03.293617+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:46:03.293617+0000","last_undegraded":"2024-09-13T14:46:03.293617+0000","last_fullsized":"2024-09-13T14:46:03.293617+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":972,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293036+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:46:03.293036+0000","last_peered":"2024-09-13T14:46:03.293036+0000","last_clean":"2024-09-13T14:46:03.293036+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:46:03.293036+0000","last_undegraded":"2024-09-13T14:46:03.293036+0000","last_fullsized":"2024-09-13T14:46:03.293036+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292410+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:46:03.292410+0000","last_peered":"2024-09-13T14:46:03.292410+0000","last_clean":"2024-09-13T14:46:03.292410+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:46:03.292410+0000","last_undegraded":"2024-09-13T14:46:03.292410+0000","last_fullsized":"2024-09-13T14:46:03.292410+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":560,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338120+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:46:13.338120+0000","last_peered":"2024-09-13T14:46:13.338120+0000","last_clean":"2024-09-13T14:46:13.338120+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:46:13.338120+0000","last_undegraded":"2024-09-13T14:46:13.338120+0000","last_fullsized":"2024-09-13T14:46:13.338120+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"357'8166","reported_seq":17568,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:18.685681+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:46:18.685681+0000","last_peered":"2024-09-13T14:46:18.685681+0000","last_clean":"2024-09-13T14:46:18.685681+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:46:18.685681+0000","last_undegraded":"2024-09-13T14:46:18.685681+0000","last_fullsized":"2024-09-13T14:46:18.685681+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8166,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":356,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293509+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:46:03.293509+0000","last_peered":"2024-09-13T14:46:03.293509+0000","last_clean":"2024-09-13T14:46:03.293509+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:46:03.293509+0000","last_undegraded":"2024-09-13T14:46:03.293509+0000","last_fullsized":"2024-09-13T14:46:03.293509+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293209+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:46:03.293209+0000","last_peered":"2024-09-13T14:46:03.293209+0000","last_clean":"2024-09-13T14:46:03.293209+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:46:03.293209+0000","last_undegraded":"2024-09-13T14:46:03.293209+0000","last_fullsized":"2024-09-13T14:46:03.293209+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1139,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338324+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:46:13.338324+0000","last_peered":"2024-09-13T14:46:13.338324+0000","last_clean":"2024-09-13T14:46:13.338324+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:46:13.338324+0000","last_undegraded":"2024-09-13T14:46:13.338324+0000","last_fullsized":"2024-09-13T14:46:13.338324+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":541,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337859+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:46:13.337859+0000","last_peered":"2024-09-13T14:46:13.337859+0000","last_clean":"2024-09-13T14:46:13.337859+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:46:13.337859+0000","last_undegraded":"2024-09-13T14:46:13.337859+0000","last_fullsized":"2024-09-13T14:46:13.337859+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"357'8004","reported_seq":8125,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:15.184667+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:46:15.184667+0000","last_peered":"2024-09-13T14:46:15.184667+0000","last_clean":"2024-09-13T14:46:15.184667+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:46:15.184667+0000","last_undegraded":"2024-09-13T14:46:15.184667+0000","last_fullsized":"2024-09-13T14:46:15.184667+0000","mapping_epoch":303,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8004,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":356,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293207+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:46:03.293207+0000","last_peered":"2024-09-13T14:46:03.293207+0000","last_clean":"2024-09-13T14:46:03.293207+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:46:03.293207+0000","last_undegraded":"2024-09-13T14:46:03.293207+0000","last_fullsized":"2024-09-13T14:46:03.293207+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":550,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338222+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:46:13.338222+0000","last_peered":"2024-09-13T14:46:13.338222+0000","last_clean":"2024-09-13T14:46:13.338222+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:46:13.338222+0000","last_undegraded":"2024-09-13T14:46:13.338222+0000","last_fullsized":"2024-09-13T14:46:13.338222+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":751,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.382029+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:46:03.382029+0000","last_peered":"2024-09-13T14:46:03.382029+0000","last_clean":"2024-09-13T14:46:03.382029+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:46:03.382029+0000","last_undegraded":"2024-09-13T14:46:03.382029+0000","last_fullsized":"2024-09-13T14:46:03.382029+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":585,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325066+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:46:13.325066+0000","last_peered":"2024-09-13T14:46:13.325066+0000","last_clean":"2024-09-13T14:46:13.325066+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:46:13.325066+0000","last_undegraded":"2024-09-13T14:46:13.325066+0000","last_fullsized":"2024-09-13T14:46:13.325066+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"357'8418","reported_seq":8540,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:18.966516+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:46:18.966516+0000","last_peered":"2024-09-13T14:46:18.966516+0000","last_clean":"2024-09-13T14:46:18.966516+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:46:18.966516+0000","last_undegraded":"2024-09-13T14:46:18.966516+0000","last_fullsized":"2024-09-13T14:46:18.966516+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8418,"log_dups_size":0,"ondisk_log_size":8418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214433792,"num_objects":3272,"num_object_clones":0,"num_object_copies":9816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8418,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292718+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:46:03.292718+0000","last_peered":"2024-09-13T14:46:03.292718+0000","last_clean":"2024-09-13T14:46:03.292718+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:46:03.292718+0000","last_undegraded":"2024-09-13T14:46:03.292718+0000","last_fullsized":"2024-09-13T14:46:03.292718+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":586,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.327435+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:46:13.327435+0000","last_peered":"2024-09-13T14:46:13.327435+0000","last_clean":"2024-09-13T14:46:13.327435+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:46:13.327435+0000","last_undegraded":"2024-09-13T14:46:13.327435+0000","last_fullsized":"2024-09-13T14:46:13.327435+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":926,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338923+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:46:13.338923+0000","last_peered":"2024-09-13T14:46:13.338923+0000","last_clean":"2024-09-13T14:46:13.338923+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:46:13.338923+0000","last_undegraded":"2024-09-13T14:46:13.338923+0000","last_fullsized":"2024-09-13T14:46:13.338923+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":584,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.334868+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:46:13.334868+0000","last_peered":"2024-09-13T14:46:13.334868+0000","last_clean":"2024-09-13T14:46:13.334868+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:46:13.334868+0000","last_undegraded":"2024-09-13T14:46:13.334868+0000","last_fullsized":"2024-09-13T14:46:13.334868+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"357'8316","reported_seq":19684,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:17.505284+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:46:17.505284+0000","last_peered":"2024-09-13T14:46:17.505284+0000","last_clean":"2024-09-13T14:46:17.505284+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:46:17.505284+0000","last_undegraded":"2024-09-13T14:46:17.505284+0000","last_fullsized":"2024-09-13T14:46:17.505284+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8316,"log_dups_size":0,"ondisk_log_size":8316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518616,"num_objects":3198,"num_object_clones":0,"num_object_copies":9594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3198,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8316,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":437,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326780+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:46:13.326780+0000","last_peered":"2024-09-13T14:46:13.326780+0000","last_clean":"2024-09-13T14:46:13.326780+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:46:13.326780+0000","last_undegraded":"2024-09-13T14:46:13.326780+0000","last_fullsized":"2024-09-13T14:46:13.326780+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":621,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326994+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:46:13.326994+0000","last_peered":"2024-09-13T14:46:13.326994+0000","last_clean":"2024-09-13T14:46:13.326994+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:46:13.326994+0000","last_undegraded":"2024-09-13T14:46:13.326994+0000","last_fullsized":"2024-09-13T14:46:13.326994+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":775,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293651+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:46:03.293651+0000","last_peered":"2024-09-13T14:46:03.293651+0000","last_clean":"2024-09-13T14:46:03.293651+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:46:03.293651+0000","last_undegraded":"2024-09-13T14:46:03.293651+0000","last_fullsized":"2024-09-13T14:46:03.293651+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.291940+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:46:03.291940+0000","last_peered":"2024-09-13T14:46:03.291940+0000","last_clean":"2024-09-13T14:46:03.291940+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:46:03.291940+0000","last_undegraded":"2024-09-13T14:46:03.291940+0000","last_fullsized":"2024-09-13T14:46:03.291940+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292471+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:46:03.292471+0000","last_peered":"2024-09-13T14:46:03.292471+0000","last_clean":"2024-09-13T14:46:03.292471+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:46:03.292471+0000","last_undegraded":"2024-09-13T14:46:03.292471+0000","last_fullsized":"2024-09-13T14:46:03.292471+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":66,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292217+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:46:03.292217+0000","last_peered":"2024-09-13T14:46:03.292217+0000","last_clean":"2024-09-13T14:46:03.292217+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:46:03.292217+0000","last_undegraded":"2024-09-13T14:46:03.292217+0000","last_fullsized":"2024-09-13T14:46:03.292217+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":537,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337701+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:46:13.337701+0000","last_peered":"2024-09-13T14:46:13.337701+0000","last_clean":"2024-09-13T14:46:13.337701+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:46:13.337701+0000","last_undegraded":"2024-09-13T14:46:13.337701+0000","last_fullsized":"2024-09-13T14:46:13.337701+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":356,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.381832+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:46:03.381832+0000","last_peered":"2024-09-13T14:46:03.381832+0000","last_clean":"2024-09-13T14:46:03.381832+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:46:03.381832+0000","last_undegraded":"2024-09-13T14:46:03.381832+0000","last_fullsized":"2024-09-13T14:46:03.381832+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":322,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293654+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:46:03.293654+0000","last_peered":"2024-09-13T14:46:03.293654+0000","last_clean":"2024-09-13T14:46:03.293654+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:46:03.293654+0000","last_undegraded":"2024-09-13T14:46:03.293654+0000","last_fullsized":"2024-09-13T14:46:03.293654+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":542,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325913+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:46:13.325913+0000","last_peered":"2024-09-13T14:46:13.325913+0000","last_clean":"2024-09-13T14:46:13.325913+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:46:13.325913+0000","last_undegraded":"2024-09-13T14:46:13.325913+0000","last_fullsized":"2024-09-13T14:46:13.325913+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":594,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325466+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:46:13.325466+0000","last_peered":"2024-09-13T14:46:13.325466+0000","last_clean":"2024-09-13T14:46:13.325466+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:46:13.325466+0000","last_undegraded":"2024-09-13T14:46:13.325466+0000","last_fullsized":"2024-09-13T14:46:13.325466+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":598,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.334860+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:46:13.334860+0000","last_peered":"2024-09-13T14:46:13.334860+0000","last_clean":"2024-09-13T14:46:13.334860+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:46:13.334860+0000","last_undegraded":"2024-09-13T14:46:13.334860+0000","last_fullsized":"2024-09-13T14:46:13.334860+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":354,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293579+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:46:03.293579+0000","last_peered":"2024-09-13T14:46:03.293579+0000","last_clean":"2024-09-13T14:46:03.293579+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:46:03.293579+0000","last_undegraded":"2024-09-13T14:46:03.293579+0000","last_fullsized":"2024-09-13T14:46:03.293579+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":616,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.335025+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:46:13.335025+0000","last_peered":"2024-09-13T14:46:13.335025+0000","last_clean":"2024-09-13T14:46:13.335025+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:46:13.335025+0000","last_undegraded":"2024-09-13T14:46:13.335025+0000","last_fullsized":"2024-09-13T14:46:13.335025+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":586,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.327160+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:46:13.327160+0000","last_peered":"2024-09-13T14:46:13.327160+0000","last_clean":"2024-09-13T14:46:13.327160+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:46:13.327160+0000","last_undegraded":"2024-09-13T14:46:13.327160+0000","last_fullsized":"2024-09-13T14:46:13.327160+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":562,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337486+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:46:13.337486+0000","last_peered":"2024-09-13T14:46:13.337486+0000","last_clean":"2024-09-13T14:46:13.337486+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:46:13.337486+0000","last_undegraded":"2024-09-13T14:46:13.337486+0000","last_fullsized":"2024-09-13T14:46:13.337486+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":611,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326091+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:46:13.326091+0000","last_peered":"2024-09-13T14:46:13.326091+0000","last_clean":"2024-09-13T14:46:13.326091+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:46:13.326091+0000","last_undegraded":"2024-09-13T14:46:13.326091+0000","last_fullsized":"2024-09-13T14:46:13.326091+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":55,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.331177+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:46:03.331177+0000","last_peered":"2024-09-13T14:46:03.331177+0000","last_clean":"2024-09-13T14:46:03.331177+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:46:03.331177+0000","last_undegraded":"2024-09-13T14:46:03.331177+0000","last_fullsized":"2024-09-13T14:46:03.331177+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":508,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338483+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:46:13.338483+0000","last_peered":"2024-09-13T14:46:13.338483+0000","last_clean":"2024-09-13T14:46:13.338483+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:46:13.338483+0000","last_undegraded":"2024-09-13T14:46:13.338483+0000","last_fullsized":"2024-09-13T14:46:13.338483+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":589,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.324949+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:46:13.324949+0000","last_peered":"2024-09-13T14:46:13.324949+0000","last_clean":"2024-09-13T14:46:13.324949+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:46:13.324949+0000","last_undegraded":"2024-09-13T14:46:13.324949+0000","last_fullsized":"2024-09-13T14:46:13.324949+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.292343+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:46:03.292343+0000","last_peered":"2024-09-13T14:46:03.292343+0000","last_clean":"2024-09-13T14:46:03.292343+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:46:03.292343+0000","last_undegraded":"2024-09-13T14:46:03.292343+0000","last_fullsized":"2024-09-13T14:46:03.292343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":322,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293520+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:46:03.293520+0000","last_peered":"2024-09-13T14:46:03.293520+0000","last_clean":"2024-09-13T14:46:03.293520+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:46:03.293520+0000","last_undegraded":"2024-09-13T14:46:03.293520+0000","last_fullsized":"2024-09-13T14:46:03.293520+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":55,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.331091+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:46:03.331091+0000","last_peered":"2024-09-13T14:46:03.331091+0000","last_clean":"2024-09-13T14:46:03.331091+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:46:03.331091+0000","last_undegraded":"2024-09-13T14:46:03.331091+0000","last_fullsized":"2024-09-13T14:46:03.331091+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":288,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.293424+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:46:03.293424+0000","last_peered":"2024-09-13T14:46:03.293424+0000","last_clean":"2024-09-13T14:46:03.293424+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:46:03.293424+0000","last_undegraded":"2024-09-13T14:46:03.293424+0000","last_fullsized":"2024-09-13T14:46:03.293424+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":608,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326257+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:46:13.326257+0000","last_peered":"2024-09-13T14:46:13.326257+0000","last_clean":"2024-09-13T14:46:13.326257+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:46:13.326257+0000","last_undegraded":"2024-09-13T14:46:13.326257+0000","last_fullsized":"2024-09-13T14:46:13.326257+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":595,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.326560+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:46:13.326560+0000","last_peered":"2024-09-13T14:46:13.326560+0000","last_clean":"2024-09-13T14:46:13.326560+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:46:13.326560+0000","last_undegraded":"2024-09-13T14:46:13.326560+0000","last_fullsized":"2024-09-13T14:46:13.326560+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-09-13T14:46:03.291688+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:46:03.291688+0000","last_peered":"2024-09-13T14:46:03.291688+0000","last_clean":"2024-09-13T14:46:03.291688+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:46:03.291688+0000","last_undegraded":"2024-09-13T14:46:03.291688+0000","last_fullsized":"2024-09-13T14:46:03.291688+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":599,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.325342+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:46:13.325342+0000","last_peered":"2024-09-13T14:46:13.325342+0000","last_clean":"2024-09-13T14:46:13.325342+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:46:13.325342+0000","last_undegraded":"2024-09-13T14:46:13.325342+0000","last_fullsized":"2024-09-13T14:46:13.325342+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3393781784,"num_objects":51786,"num_object_clones":0,"num_object_copies":155358,"num_objects_missing_on_primary":2224,"num_objects_missing":2224,"num_objects_degraded":35606,"num_objects_misplaced":3176,"num_objects_unfound":0,"num_objects_dirty":51786,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130892,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24715,"num_bytes_recovered":1559298048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9504108544,"data_stored":9504096328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122192,"ondisk_log_size":122192,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397922,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1942220,"kb_used_data":1050372,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91802932,"statfs":{"total":95995035648,"available":94006202368,"internally_reserved":0,"allocated":1075580928,"data_stored":1073686590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594150,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1981560,"kb_used_data":1395824,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91763592,"statfs":{"total":95995035648,"available":93965918208,"internally_reserved":0,"allocated":1429323776,"data_stored":1427449192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665604,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589752,"kb_used_data":257200,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93155400,"statfs":{"total":95995035648,"available":95391129600,"internally_reserved":0,"allocated":263372800,"data_stored":261494547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986605,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1661192,"kb_used_data":769660,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":92083960,"statfs":{"total":95995035648,"available":94293975040,"internally_reserved":0,"allocated":788131840,"data_stored":786245600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697165,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2366392,"kb_used_data":1620716,"kb_used_omap":4,"kb_used_meta":745659,"kb_avail":91378760,"statfs":{"total":95995035648,"available":93571850240,"internally_reserved":0,"allocated":1659613184,"data_stored":1657739084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":763554891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860692,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2162684,"kb_used_data":1537268,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91582468,"statfs":{"total":95995035648,"available":93780447232,"internally_reserved":0,"allocated":1574162432,"data_stored":1572280678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020363,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1762160,"kb_used_data":1462760,"kb_used_omap":9,"kb_used_meta":299382,"kb_avail":91982992,"statfs":{"total":95995035648,"available":94190583808,"internally_reserved":0,"allocated":1497866240,"data_stored":1495995613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":306568005},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187754,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1863344,"kb_used_data":1248680,"kb_used_omap":7,"kb_used_meta":614648,"kb_avail":91881808,"statfs":{"total":95995035648,"available":94086971392,"internally_reserved":0,"allocated":1278648320,"data_stored":1276777959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":629399883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1268842496,"data_stored":1268842496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1489043456,"data_stored":1489043456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1569263616,"data_stored":1569259544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1652293632,"data_stored":1652293632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":780861440,"data_stored":780861440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":257298432,"data_stored":257294360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1421148160,"data_stored":1421148160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1065357312,"data_stored":1065353240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:21.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:21.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:21.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:21.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:21.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:21.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:21.499 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:21 smithi189 ceph-mon[90992]: from='client.36495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:21.499 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3756571053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:21.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:21.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:21.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:21.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:21.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:21 smithi017 ceph-mon[103001]: from='client.36495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:21.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3756571053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:21.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:21 smithi017 ceph-mon[107556]: from='client.36495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:21.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3756571053' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:21.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:21.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:21.763 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:21.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:21.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:21.978 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:21.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:22.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:22.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:22.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:22.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:22.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:22.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:22.429 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:22 smithi189 ceph-mon[90992]: pgmap v1123: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35555/154356 objects degraded (23.034%); 3125/154356 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:22.438 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:46:22.438 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:46:22.438 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:46:22.438 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:46:22.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:22.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:22.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:22 smithi017 ceph-mon[103001]: pgmap v1123: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35555/154356 objects degraded (23.034%); 3125/154356 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:22 smithi017 ceph-mon[107556]: pgmap v1123: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 35555/154356 objects degraded (23.034%); 3125/154356 objects misplaced (2.025%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:22.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:22.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:22.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:22.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:22.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:22.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:23.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:23.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:46:23.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:23.490 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:46:23.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:23.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:23.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:23.759 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:23.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:23.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:23.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:23.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:24.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:24.016 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:24.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:24.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:24.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:24.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:24.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:24.339 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:46:24.360 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:24.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:24.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:24.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[103001]: pgmap v1124: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 35489/153948 objects degraded (23.053%); 3125/153948 objects misplaced (2.030%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:24.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35489/153948 objects degraded (23.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:24.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1254221339' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.440 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[107556]: pgmap v1124: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 35489/153948 objects degraded (23.053%); 3125/153948 objects misplaced (2.030%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:24.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35489/153948 objects degraded (23.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:24.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1254221339' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:24 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:24.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:24.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:24.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:24.678 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:24 smithi189 ceph-mon[90992]: pgmap v1124: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 35489/153948 objects degraded (23.053%); 3125/153948 objects misplaced (2.030%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:24.679 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35489/153948 objects degraded (23.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:24.679 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:24 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1254221339' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.679 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:24 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:24.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:24.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:24.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:24.990 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:25.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:25.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:25.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:25.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:25.250 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:46:25.250 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:46:25.250 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:46:25.251 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:46:25.251 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:46:25.264 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:25.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:25.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:25.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[103001]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:46:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:25 smithi017 ceph-mon[107556]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:46:25.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:25 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:25.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:25 smithi189 ceph-mon[90992]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:46:25.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:25.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:25.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:25.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:25.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:25.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:25.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:25.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:46:25.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:26.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:26.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:26.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:26.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:26.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:26.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:26.283 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:46:26.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:26.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:26.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:26.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:26.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:26 smithi017 ceph-mon[103001]: pgmap v1126: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 35438/152997 objects degraded (23.163%); 3076/152997 objects misplaced (2.010%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:26.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:26 smithi017 ceph-mon[107556]: pgmap v1126: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 35438/152997 objects degraded (23.163%); 3076/152997 objects misplaced (2.010%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:26.647 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:26.741 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:26 smithi189 ceph-mon[90992]: pgmap v1126: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 35438/152997 objects degraded (23.163%); 3076/152997 objects misplaced (2.010%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:46:26.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:26.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:26.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:26.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:27.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:27.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:27.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:27.148 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:27.249 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:27.265 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:46:27.265 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:27.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:27.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:27.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:46:27.284 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:27.307 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:46:27.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:27.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:46:27.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:27.346 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:46:27.346 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:27.367 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:46:27.367 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:27.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:27.391 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:46:27.391 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:27.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:27.416 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:46:27.416 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:27.416 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:27] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:46:27.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:27.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:27 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/501967631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:27.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:27.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:27 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/501967631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:27.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:27 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/501967631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:27.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:27.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:46:27.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:27.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:27.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:27.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:27.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:27.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:27.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:28.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.122 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:28.122 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:28.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:28.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:28.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:28.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:28.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:28.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:28.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:28 smithi189 ceph-mon[90992]: pgmap v1127: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 35332/152343 objects degraded (23.192%); 3076/152343 objects misplaced (2.019%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:28.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:28.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:28.631 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:28.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:28 smithi017 ceph-mon[103001]: pgmap v1127: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 35332/152343 objects degraded (23.192%); 3076/152343 objects misplaced (2.019%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:28 smithi017 ceph-mon[107556]: pgmap v1127: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 35332/152343 objects degraded (23.192%); 3076/152343 objects misplaced (2.019%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:28.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:28.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:28.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:46:28.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:28.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:29.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:29.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:29.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:29.412 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:29.426 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:29.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:29.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35332/152343 objects degraded (23.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:29.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35332/152343 objects degraded (23.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35332/152343 objects degraded (23.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:29.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:29.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:29.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:29.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:29.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:30.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:30.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:30.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:30.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:30.280 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:30.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:30.378 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:30.379 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:46:30.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:30.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:30 smithi189 ceph-mon[90992]: pgmap v1128: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35332/151440 objects degraded (23.331%); 3076/151440 objects misplaced (2.031%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:30.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:30.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:30.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:30.568 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:30 smithi017 ceph-mon[103001]: pgmap v1128: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35332/151440 objects degraded (23.331%); 3076/151440 objects misplaced (2.031%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:30.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:30 smithi017 ceph-mon[107556]: pgmap v1128: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35332/151440 objects degraded (23.331%); 3076/151440 objects misplaced (2.031%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:30.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:30.809 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:30.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:30.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:30.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:31.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:31.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:31.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:31.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:31.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:31.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:31.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:31.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:31.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:31.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:31 smithi017 ceph-mon[103001]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:31.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:31 smithi017 ceph-mon[107556]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:31.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:31.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:31.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:31 smithi189 ceph-mon[90992]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:31.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:31.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:31.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:31.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:31.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:31.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:32.003 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1128,"stamp":"2024-09-13T14:46:29.241415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3311045302,"num_objects":50480,"num_object_clones":0,"num_object_copies":151440,"num_objects_missing_on_primary":2122,"num_objects_missing":2122,"num_objects_degraded":35332,"num_objects_misplaced":3076,"num_objects_unfound":0,"num_objects_dirty":50480,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":133173,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24958,"num_bytes_recovered":1571653789,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124858,"ondisk_log_size":124858,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14128212,"kb_used_data":9141392,"kb_used_omap":44,"kb_used_meta":4986707,"kb_avail":735833004,"statfs":{"total":767960285184,"available":753492996096,"internally_reserved":0,"allocated":9360785408,"data_stored":9345692375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5106388215},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,2,1,1,0,1,3,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-87425024,"num_objects":-1334,"num_object_clones":0,"num_object_copies":-4002,"num_objects_missing_on_primary":-102,"num_objects_missing":-102,"num_objects_degraded":-274,"num_objects_misplaced":-100,"num_objects_unfound":0,"num_objects_dirty":-1334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1334,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":12189696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003163"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":600,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.436388+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:46:24.436388+0000","last_peered":"2024-09-13T14:46:24.436388+0000","last_clean":"2024-09-13T14:46:24.436388+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:46:24.436388+0000","last_undegraded":"2024-09-13T14:46:24.436388+0000","last_fullsized":"2024-09-13T14:46:24.436388+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":356,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337694+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:46:13.337694+0000","last_peered":"2024-09-13T14:46:13.337694+0000","last_clean":"2024-09-13T14:46:13.337694+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:46:13.337694+0000","last_undegraded":"2024-09-13T14:46:13.337694+0000","last_fullsized":"2024-09-13T14:46:13.337694+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":285,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320117+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:46:13.320117+0000","last_peered":"2024-09-13T14:46:13.320117+0000","last_clean":"2024-09-13T14:46:13.320117+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:46:13.320117+0000","last_undegraded":"2024-09-13T14:46:13.320117+0000","last_fullsized":"2024-09-13T14:46:13.320117+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":585,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337175+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:46:24.337175+0000","last_peered":"2024-09-13T14:46:24.337175+0000","last_clean":"2024-09-13T14:46:24.337175+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:46:24.337175+0000","last_undegraded":"2024-09-13T14:46:24.337175+0000","last_fullsized":"2024-09-13T14:46:24.337175+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":586,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337556+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:46:24.337556+0000","last_peered":"2024-09-13T14:46:24.337556+0000","last_clean":"2024-09-13T14:46:24.337556+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:46:24.337556+0000","last_undegraded":"2024-09-13T14:46:24.337556+0000","last_fullsized":"2024-09-13T14:46:24.337556+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":325,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337755+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:46:13.337755+0000","last_peered":"2024-09-13T14:46:13.337755+0000","last_clean":"2024-09-13T14:46:13.337755+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:46:13.337755+0000","last_undegraded":"2024-09-13T14:46:13.337755+0000","last_fullsized":"2024-09-13T14:46:13.337755+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":600,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.438180+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:46:24.438180+0000","last_peered":"2024-09-13T14:46:24.438180+0000","last_clean":"2024-09-13T14:46:24.438180+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:46:24.438180+0000","last_undegraded":"2024-09-13T14:46:24.438180+0000","last_fullsized":"2024-09-13T14:46:24.438180+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":347,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319347+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:46:13.319347+0000","last_peered":"2024-09-13T14:46:13.319347+0000","last_clean":"2024-09-13T14:46:13.319347+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:46:13.319347+0000","last_undegraded":"2024-09-13T14:46:13.319347+0000","last_fullsized":"2024-09-13T14:46:13.319347+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"358'8222","reported_seq":16070,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:26.956397+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:46:26.956397+0000","last_peered":"2024-09-13T14:46:26.956397+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:46:26.956397+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8222,"log_dups_size":0,"ondisk_log_size":8222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201064448,"num_objects":3068,"num_object_clones":0,"num_object_copies":9204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5051,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8222,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2577,"num_bytes_recovered":168886272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":2084}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":351,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.338061+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:46:13.338061+0000","last_peered":"2024-09-13T14:46:13.338061+0000","last_clean":"2024-09-13T14:46:13.338061+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:46:13.338061+0000","last_undegraded":"2024-09-13T14:46:13.338061+0000","last_fullsized":"2024-09-13T14:46:13.338061+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319222+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:46:13.319222+0000","last_peered":"2024-09-13T14:46:13.319222+0000","last_clean":"2024-09-13T14:46:13.319222+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:46:13.319222+0000","last_undegraded":"2024-09-13T14:46:13.319222+0000","last_fullsized":"2024-09-13T14:46:13.319222+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"358'8338","reported_seq":17893,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:27.876975+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:46:27.876975+0000","last_peered":"2024-09-13T14:46:27.876975+0000","last_clean":"2024-09-13T14:46:27.876975+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:46:27.876975+0000","last_undegraded":"2024-09-13T14:46:27.876975+0000","last_fullsized":"2024-09-13T14:46:27.876975+0000","mapping_epoch":326,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206438400,"num_objects":3150,"num_object_clones":0,"num_object_copies":9450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8338,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":369,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337974+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:46:13.337974+0000","last_peered":"2024-09-13T14:46:13.337974+0000","last_clean":"2024-09-13T14:46:13.337974+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:46:13.337974+0000","last_undegraded":"2024-09-13T14:46:13.337974+0000","last_fullsized":"2024-09-13T14:46:13.337974+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317288+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:46:13.317288+0000","last_peered":"2024-09-13T14:46:13.317288+0000","last_clean":"2024-09-13T14:46:13.317288+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:46:13.317288+0000","last_undegraded":"2024-09-13T14:46:13.317288+0000","last_fullsized":"2024-09-13T14:46:13.317288+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"358'8586","reported_seq":8710,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:28.784995+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:46:28.784995+0000","last_peered":"2024-09-13T14:46:28.784995+0000","last_clean":"2024-09-13T14:46:28.784995+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:46:28.784995+0000","last_undegraded":"2024-09-13T14:46:28.784995+0000","last_fullsized":"2024-09-13T14:46:28.784995+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8586,"log_dups_size":0,"ondisk_log_size":8586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210501632,"num_objects":3212,"num_object_clones":0,"num_object_copies":9636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8586,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.340209+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:46:13.340209+0000","last_peered":"2024-09-13T14:46:13.340209+0000","last_clean":"2024-09-13T14:46:13.340209+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:46:13.340209+0000","last_undegraded":"2024-09-13T14:46:13.340209+0000","last_fullsized":"2024-09-13T14:46:13.340209+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":338,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319547+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:46:13.319547+0000","last_peered":"2024-09-13T14:46:13.319547+0000","last_clean":"2024-09-13T14:46:13.319547+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:46:13.319547+0000","last_undegraded":"2024-09-13T14:46:13.319547+0000","last_fullsized":"2024-09-13T14:46:13.319547+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"358'8357","reported_seq":8481,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.753195+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:46:24.753195+0000","last_peered":"2024-09-13T14:46:24.753195+0000","last_clean":"2024-09-13T14:46:24.753195+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:46:24.753195+0000","last_undegraded":"2024-09-13T14:46:24.753195+0000","last_fullsized":"2024-09-13T14:46:24.753195+0000","mapping_epoch":303,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208994304,"num_objects":3189,"num_object_clones":0,"num_object_copies":9567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8357,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":286,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319623+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:46:13.319623+0000","last_peered":"2024-09-13T14:46:13.319623+0000","last_clean":"2024-09-13T14:46:13.319623+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:46:13.319623+0000","last_undegraded":"2024-09-13T14:46:13.319623+0000","last_fullsized":"2024-09-13T14:46:13.319623+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":612,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337480+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:46:24.337480+0000","last_peered":"2024-09-13T14:46:24.337480+0000","last_clean":"2024-09-13T14:46:24.337480+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:46:24.337480+0000","last_undegraded":"2024-09-13T14:46:24.337480+0000","last_fullsized":"2024-09-13T14:46:24.337480+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"358'8317","reported_seq":8438,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:25.463698+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:46:25.463698+0000","last_peered":"2024-09-13T14:46:25.463698+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:46:25.463698+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206110720,"num_objects":3145,"num_object_clones":0,"num_object_copies":9435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8317,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2488}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":469,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337321+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:46:13.337321+0000","last_peered":"2024-09-13T14:46:13.337321+0000","last_clean":"2024-09-13T14:46:13.337321+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:46:13.337321+0000","last_undegraded":"2024-09-13T14:46:13.337321+0000","last_fullsized":"2024-09-13T14:46:13.337321+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":609,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435269+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:46:24.435269+0000","last_peered":"2024-09-13T14:46:24.435269+0000","last_clean":"2024-09-13T14:46:24.435269+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:46:24.435269+0000","last_undegraded":"2024-09-13T14:46:24.435269+0000","last_fullsized":"2024-09-13T14:46:24.435269+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317426+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:46:13.317426+0000","last_peered":"2024-09-13T14:46:13.317426+0000","last_clean":"2024-09-13T14:46:13.317426+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:46:13.317426+0000","last_undegraded":"2024-09-13T14:46:13.317426+0000","last_fullsized":"2024-09-13T14:46:13.317426+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"358'7965","reported_seq":8089,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:25.037645+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:46:25.037645+0000","last_peered":"2024-09-13T14:46:25.037645+0000","last_clean":"2024-09-13T14:46:25.037645+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:46:25.037645+0000","last_undegraded":"2024-09-13T14:46:25.037645+0000","last_fullsized":"2024-09-13T14:46:25.037645+0000","mapping_epoch":303,"log_start":"306'400","ondisk_log_start":"306'400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205062144,"num_objects":3129,"num_object_clones":0,"num_object_copies":9387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7965,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":173,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319315+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:46:13.319315+0000","last_peered":"2024-09-13T14:46:13.319315+0000","last_clean":"2024-09-13T14:46:13.319315+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:46:13.319315+0000","last_undegraded":"2024-09-13T14:46:13.319315+0000","last_fullsized":"2024-09-13T14:46:13.319315+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":359,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319419+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:46:13.319419+0000","last_peered":"2024-09-13T14:46:13.319419+0000","last_clean":"2024-09-13T14:46:13.319419+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:46:13.319419+0000","last_undegraded":"2024-09-13T14:46:13.319419+0000","last_fullsized":"2024-09-13T14:46:13.319419+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":71,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317773+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:46:13.317773+0000","last_peered":"2024-09-13T14:46:13.317773+0000","last_clean":"2024-09-13T14:46:13.317773+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:46:13.317773+0000","last_undegraded":"2024-09-13T14:46:13.317773+0000","last_fullsized":"2024-09-13T14:46:13.317773+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"358'8232","reported_seq":15534,"reported_epoch":358,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:24.855247+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:46:24.855247+0000","last_peered":"2024-09-13T14:46:24.855247+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:46:24.855247+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7332,"log_dups_size":0,"ondisk_log_size":7332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208404480,"num_objects":3180,"num_object_clones":0,"num_object_copies":9540,"num_objects_missing_on_primary":2122,"num_objects_missing":2122,"num_objects_degraded":2497,"num_objects_misplaced":3076,"num_objects_unfound":0,"num_objects_dirty":3180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8232,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2527,"num_bytes_recovered":165609472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2122},{"shards":"0,3,4","objects":954},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317144+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:46:13.317144+0000","last_peered":"2024-09-13T14:46:13.317144+0000","last_clean":"2024-09-13T14:46:13.317144+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:46:13.317144+0000","last_undegraded":"2024-09-13T14:46:13.317144+0000","last_fullsized":"2024-09-13T14:46:13.317144+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.318089+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:46:13.318089+0000","last_peered":"2024-09-13T14:46:13.318089+0000","last_clean":"2024-09-13T14:46:13.318089+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:46:13.318089+0000","last_undegraded":"2024-09-13T14:46:13.318089+0000","last_fullsized":"2024-09-13T14:46:13.318089+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"358'8053","reported_seq":8177,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:24.905872+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:46:24.905872+0000","last_peered":"2024-09-13T14:46:24.905872+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:46:24.905872+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203489280,"num_objects":3105,"num_object_clones":0,"num_object_copies":9315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8053,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2361}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":166,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319589+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:46:13.319589+0000","last_peered":"2024-09-13T14:46:13.319589+0000","last_clean":"2024-09-13T14:46:13.319589+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:46:13.319589+0000","last_undegraded":"2024-09-13T14:46:13.319589+0000","last_fullsized":"2024-09-13T14:46:13.319589+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":602,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.444704+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:46:24.444704+0000","last_peered":"2024-09-13T14:46:24.444704+0000","last_clean":"2024-09-13T14:46:24.444704+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:46:24.444704+0000","last_undegraded":"2024-09-13T14:46:24.444704+0000","last_fullsized":"2024-09-13T14:46:24.444704+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":564,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423797+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:46:24.423797+0000","last_peered":"2024-09-13T14:46:24.423797+0000","last_clean":"2024-09-13T14:46:24.423797+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:46:24.423797+0000","last_undegraded":"2024-09-13T14:46:24.423797+0000","last_fullsized":"2024-09-13T14:46:24.423797+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"358'8111","reported_seq":8181,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:27.883608+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:46:27.883608+0000","last_peered":"2024-09-13T14:46:27.883608+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:46:27.883608+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203751424,"num_objects":3109,"num_object_clones":0,"num_object_copies":9327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8111,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2387}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":460,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337554+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:46:13.337554+0000","last_peered":"2024-09-13T14:46:13.337554+0000","last_clean":"2024-09-13T14:46:13.337554+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:46:13.337554+0000","last_undegraded":"2024-09-13T14:46:13.337554+0000","last_fullsized":"2024-09-13T14:46:13.337554+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":882,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319934+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:46:13.319934+0000","last_peered":"2024-09-13T14:46:13.319934+0000","last_clean":"2024-09-13T14:46:13.319934+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:46:13.319934+0000","last_undegraded":"2024-09-13T14:46:13.319934+0000","last_fullsized":"2024-09-13T14:46:13.319934+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":555,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435222+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:46:24.435222+0000","last_peered":"2024-09-13T14:46:24.435222+0000","last_clean":"2024-09-13T14:46:24.435222+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:46:24.435222+0000","last_undegraded":"2024-09-13T14:46:24.435222+0000","last_fullsized":"2024-09-13T14:46:24.435222+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320316+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:46:13.320316+0000","last_peered":"2024-09-13T14:46:13.320316+0000","last_clean":"2024-09-13T14:46:13.320316+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:46:13.320316+0000","last_undegraded":"2024-09-13T14:46:13.320316+0000","last_fullsized":"2024-09-13T14:46:13.320316+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"358'8371","reported_seq":8493,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:24.895121+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:46:24.895121+0000","last_peered":"2024-09-13T14:46:24.895121+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:46:24.895121+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208207872,"num_objects":3177,"num_object_clones":0,"num_object_copies":9531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8371,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2503}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":409,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423734+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:46:24.423734+0000","last_peered":"2024-09-13T14:46:24.423734+0000","last_clean":"2024-09-13T14:46:24.423734+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:46:24.423734+0000","last_undegraded":"2024-09-13T14:46:24.423734+0000","last_fullsized":"2024-09-13T14:46:24.423734+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":748,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337463+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:46:13.337463+0000","last_peered":"2024-09-13T14:46:13.337463+0000","last_clean":"2024-09-13T14:46:13.337463+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:46:13.337463+0000","last_undegraded":"2024-09-13T14:46:13.337463+0000","last_fullsized":"2024-09-13T14:46:13.337463+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":604,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.336980+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:46:24.336980+0000","last_peered":"2024-09-13T14:46:24.336980+0000","last_clean":"2024-09-13T14:46:24.336980+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:46:24.336980+0000","last_undegraded":"2024-09-13T14:46:24.336980+0000","last_fullsized":"2024-09-13T14:46:24.336980+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":609,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337667+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:46:24.337667+0000","last_peered":"2024-09-13T14:46:24.337667+0000","last_clean":"2024-09-13T14:46:24.337667+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:46:24.337667+0000","last_undegraded":"2024-09-13T14:46:24.337667+0000","last_fullsized":"2024-09-13T14:46:24.337667+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"358'8358","reported_seq":18023,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:25.548821+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:46:25.548821+0000","last_peered":"2024-09-13T14:46:25.548821+0000","last_clean":"2024-09-13T14:46:25.548821+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:46:25.548821+0000","last_undegraded":"2024-09-13T14:46:25.548821+0000","last_fullsized":"2024-09-13T14:46:25.548821+0000","mapping_epoch":330,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8358,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":489,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.337574+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:46:13.337574+0000","last_peered":"2024-09-13T14:46:13.337574+0000","last_clean":"2024-09-13T14:46:13.337574+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:46:13.337574+0000","last_undegraded":"2024-09-13T14:46:13.337574+0000","last_fullsized":"2024-09-13T14:46:13.337574+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":573,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435183+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:46:24.435183+0000","last_peered":"2024-09-13T14:46:24.435183+0000","last_clean":"2024-09-13T14:46:24.435183+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:46:24.435183+0000","last_undegraded":"2024-09-13T14:46:24.435183+0000","last_fullsized":"2024-09-13T14:46:24.435183+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":541,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423425+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:46:24.423425+0000","last_peered":"2024-09-13T14:46:24.423425+0000","last_clean":"2024-09-13T14:46:24.423425+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:46:24.423425+0000","last_undegraded":"2024-09-13T14:46:24.423425+0000","last_fullsized":"2024-09-13T14:46:24.423425+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320265+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:46:13.320265+0000","last_peered":"2024-09-13T14:46:13.320265+0000","last_clean":"2024-09-13T14:46:13.320265+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:46:13.320265+0000","last_undegraded":"2024-09-13T14:46:13.320265+0000","last_fullsized":"2024-09-13T14:46:13.320265+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"358'8072","reported_seq":8195,"reported_epoch":358,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:25.537175+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:46:25.537175+0000","last_peered":"2024-09-13T14:46:25.537175+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:46:25.537175+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8072,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2351}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":166,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319522+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:46:13.319522+0000","last_peered":"2024-09-13T14:46:13.319522+0000","last_clean":"2024-09-13T14:46:13.319522+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:46:13.319522+0000","last_undegraded":"2024-09-13T14:46:13.319522+0000","last_fullsized":"2024-09-13T14:46:13.319522+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":973,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319657+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:46:13.319657+0000","last_peered":"2024-09-13T14:46:13.319657+0000","last_clean":"2024-09-13T14:46:13.319657+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:46:13.319657+0000","last_undegraded":"2024-09-13T14:46:13.319657+0000","last_fullsized":"2024-09-13T14:46:13.319657+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317550+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:46:13.317550+0000","last_peered":"2024-09-13T14:46:13.317550+0000","last_clean":"2024-09-13T14:46:13.317550+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:46:13.317550+0000","last_undegraded":"2024-09-13T14:46:13.317550+0000","last_fullsized":"2024-09-13T14:46:13.317550+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":562,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423389+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:46:24.423389+0000","last_peered":"2024-09-13T14:46:24.423389+0000","last_clean":"2024-09-13T14:46:24.423389+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:46:24.423389+0000","last_undegraded":"2024-09-13T14:46:24.423389+0000","last_fullsized":"2024-09-13T14:46:24.423389+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"358'8272","reported_seq":17676,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:28.753242+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:46:28.753242+0000","last_peered":"2024-09-13T14:46:28.753242+0000","last_clean":"2024-09-13T14:46:28.753242+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:46:28.753242+0000","last_undegraded":"2024-09-13T14:46:28.753242+0000","last_fullsized":"2024-09-13T14:46:28.753242+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8272,"log_dups_size":0,"ondisk_log_size":8272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202375168,"num_objects":3088,"num_object_clones":0,"num_object_copies":9264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8272,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":357,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320101+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:46:13.320101+0000","last_peered":"2024-09-13T14:46:13.320101+0000","last_clean":"2024-09-13T14:46:13.320101+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:46:13.320101+0000","last_undegraded":"2024-09-13T14:46:13.320101+0000","last_fullsized":"2024-09-13T14:46:13.320101+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":303,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319986+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:46:13.319986+0000","last_peered":"2024-09-13T14:46:13.319986+0000","last_clean":"2024-09-13T14:46:13.319986+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:46:13.319986+0000","last_undegraded":"2024-09-13T14:46:13.319986+0000","last_fullsized":"2024-09-13T14:46:13.319986+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1141,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423518+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:46:24.423518+0000","last_peered":"2024-09-13T14:46:24.423518+0000","last_clean":"2024-09-13T14:46:24.423518+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:46:24.423518+0000","last_undegraded":"2024-09-13T14:46:24.423518+0000","last_fullsized":"2024-09-13T14:46:24.423518+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":543,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423663+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:46:24.423663+0000","last_peered":"2024-09-13T14:46:24.423663+0000","last_clean":"2024-09-13T14:46:24.423663+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:46:24.423663+0000","last_undegraded":"2024-09-13T14:46:24.423663+0000","last_fullsized":"2024-09-13T14:46:24.423663+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"358'8081","reported_seq":8205,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:25.489235+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:46:25.489235+0000","last_peered":"2024-09-13T14:46:25.489235+0000","last_clean":"2024-09-13T14:46:25.489235+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:46:25.489235+0000","last_undegraded":"2024-09-13T14:46:25.489235+0000","last_fullsized":"2024-09-13T14:46:25.489235+0000","mapping_epoch":303,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8081,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":357,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320091+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:46:13.320091+0000","last_peered":"2024-09-13T14:46:13.320091+0000","last_clean":"2024-09-13T14:46:13.320091+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:46:13.320091+0000","last_undegraded":"2024-09-13T14:46:13.320091+0000","last_fullsized":"2024-09-13T14:46:13.320091+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":552,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423433+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:46:24.423433+0000","last_peered":"2024-09-13T14:46:24.423433+0000","last_clean":"2024-09-13T14:46:24.423433+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:46:24.423433+0000","last_undegraded":"2024-09-13T14:46:24.423433+0000","last_fullsized":"2024-09-13T14:46:24.423433+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":752,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.340208+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:46:13.340208+0000","last_peered":"2024-09-13T14:46:13.340208+0000","last_clean":"2024-09-13T14:46:13.340208+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:46:13.340208+0000","last_undegraded":"2024-09-13T14:46:13.340208+0000","last_fullsized":"2024-09-13T14:46:13.340208+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":587,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337359+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:46:24.337359+0000","last_peered":"2024-09-13T14:46:24.337359+0000","last_clean":"2024-09-13T14:46:24.337359+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:46:24.337359+0000","last_undegraded":"2024-09-13T14:46:24.337359+0000","last_fullsized":"2024-09-13T14:46:24.337359+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"358'8500","reported_seq":8624,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:28.291084+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:46:28.291084+0000","last_peered":"2024-09-13T14:46:28.291084+0000","last_clean":"2024-09-13T14:46:28.291084+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:46:28.291084+0000","last_undegraded":"2024-09-13T14:46:28.291084+0000","last_fullsized":"2024-09-13T14:46:28.291084+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8500,"log_dups_size":0,"ondisk_log_size":8500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209059840,"num_objects":3190,"num_object_clones":0,"num_object_copies":9570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8500,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317589+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:46:13.317589+0000","last_peered":"2024-09-13T14:46:13.317589+0000","last_clean":"2024-09-13T14:46:13.317589+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:46:13.317589+0000","last_undegraded":"2024-09-13T14:46:13.317589+0000","last_fullsized":"2024-09-13T14:46:13.317589+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":588,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.438485+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:46:24.438485+0000","last_peered":"2024-09-13T14:46:24.438485+0000","last_clean":"2024-09-13T14:46:24.438485+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:46:24.438485+0000","last_undegraded":"2024-09-13T14:46:24.438485+0000","last_fullsized":"2024-09-13T14:46:24.438485+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":928,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423735+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:46:24.423735+0000","last_peered":"2024-09-13T14:46:24.423735+0000","last_clean":"2024-09-13T14:46:24.423735+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:46:24.423735+0000","last_undegraded":"2024-09-13T14:46:24.423735+0000","last_fullsized":"2024-09-13T14:46:24.423735+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":586,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.444489+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:46:24.444489+0000","last_peered":"2024-09-13T14:46:24.444489+0000","last_clean":"2024-09-13T14:46:24.444489+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:46:24.444489+0000","last_undegraded":"2024-09-13T14:46:24.444489+0000","last_fullsized":"2024-09-13T14:46:24.444489+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"358'8391","reported_seq":19760,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:27.687180+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:46:27.687180+0000","last_peered":"2024-09-13T14:46:27.687180+0000","last_clean":"2024-09-13T14:46:27.687180+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:46:27.687180+0000","last_undegraded":"2024-09-13T14:46:27.687180+0000","last_fullsized":"2024-09-13T14:46:27.687180+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8391,"log_dups_size":0,"ondisk_log_size":8391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204603416,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8391,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":439,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435382+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:46:24.435382+0000","last_peered":"2024-09-13T14:46:24.435382+0000","last_clean":"2024-09-13T14:46:24.435382+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:46:24.435382+0000","last_undegraded":"2024-09-13T14:46:24.435382+0000","last_fullsized":"2024-09-13T14:46:24.435382+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":623,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337571+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:46:24.337571+0000","last_peered":"2024-09-13T14:46:24.337571+0000","last_clean":"2024-09-13T14:46:24.337571+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:46:24.337571+0000","last_undegraded":"2024-09-13T14:46:24.337571+0000","last_fullsized":"2024-09-13T14:46:24.337571+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":776,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320433+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:46:13.320433+0000","last_peered":"2024-09-13T14:46:13.320433+0000","last_clean":"2024-09-13T14:46:13.320433+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:46:13.320433+0000","last_undegraded":"2024-09-13T14:46:13.320433+0000","last_fullsized":"2024-09-13T14:46:13.320433+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317218+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:46:13.317218+0000","last_peered":"2024-09-13T14:46:13.317218+0000","last_clean":"2024-09-13T14:46:13.317218+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:46:13.317218+0000","last_undegraded":"2024-09-13T14:46:13.317218+0000","last_fullsized":"2024-09-13T14:46:13.317218+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317567+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:46:13.317567+0000","last_peered":"2024-09-13T14:46:13.317567+0000","last_clean":"2024-09-13T14:46:13.317567+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:46:13.317567+0000","last_undegraded":"2024-09-13T14:46:13.317567+0000","last_fullsized":"2024-09-13T14:46:13.317567+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":67,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317329+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:46:13.317329+0000","last_peered":"2024-09-13T14:46:13.317329+0000","last_clean":"2024-09-13T14:46:13.317329+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:46:13.317329+0000","last_undegraded":"2024-09-13T14:46:13.317329+0000","last_fullsized":"2024-09-13T14:46:13.317329+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":540,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423554+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:46:24.423554+0000","last_peered":"2024-09-13T14:46:24.423554+0000","last_clean":"2024-09-13T14:46:24.423554+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:46:24.423554+0000","last_undegraded":"2024-09-13T14:46:24.423554+0000","last_fullsized":"2024-09-13T14:46:24.423554+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":357,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.340113+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:46:13.340113+0000","last_peered":"2024-09-13T14:46:13.340113+0000","last_clean":"2024-09-13T14:46:13.340113+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:46:13.340113+0000","last_undegraded":"2024-09-13T14:46:13.340113+0000","last_fullsized":"2024-09-13T14:46:13.340113+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320194+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:46:13.320194+0000","last_peered":"2024-09-13T14:46:13.320194+0000","last_clean":"2024-09-13T14:46:13.320194+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:46:13.320194+0000","last_undegraded":"2024-09-13T14:46:13.320194+0000","last_fullsized":"2024-09-13T14:46:13.320194+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":544,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435001+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:46:24.435001+0000","last_peered":"2024-09-13T14:46:24.435001+0000","last_clean":"2024-09-13T14:46:24.435001+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:46:24.435001+0000","last_undegraded":"2024-09-13T14:46:24.435001+0000","last_fullsized":"2024-09-13T14:46:24.435001+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":596,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337247+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:46:24.337247+0000","last_peered":"2024-09-13T14:46:24.337247+0000","last_clean":"2024-09-13T14:46:24.337247+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:46:24.337247+0000","last_undegraded":"2024-09-13T14:46:24.337247+0000","last_fullsized":"2024-09-13T14:46:24.337247+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":600,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.444391+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:46:24.444391+0000","last_peered":"2024-09-13T14:46:24.444391+0000","last_clean":"2024-09-13T14:46:24.444391+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:46:24.444391+0000","last_undegraded":"2024-09-13T14:46:24.444391+0000","last_fullsized":"2024-09-13T14:46:24.444391+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":355,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320363+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:46:13.320363+0000","last_peered":"2024-09-13T14:46:13.320363+0000","last_clean":"2024-09-13T14:46:13.320363+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:46:13.320363+0000","last_undegraded":"2024-09-13T14:46:13.320363+0000","last_fullsized":"2024-09-13T14:46:13.320363+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":618,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.444587+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:46:24.444587+0000","last_peered":"2024-09-13T14:46:24.444587+0000","last_clean":"2024-09-13T14:46:24.444587+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:46:24.444587+0000","last_undegraded":"2024-09-13T14:46:24.444587+0000","last_fullsized":"2024-09-13T14:46:24.444587+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":588,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337633+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:46:24.337633+0000","last_peered":"2024-09-13T14:46:24.337633+0000","last_clean":"2024-09-13T14:46:24.337633+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:46:24.337633+0000","last_undegraded":"2024-09-13T14:46:24.337633+0000","last_fullsized":"2024-09-13T14:46:24.337633+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":564,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423346+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:46:24.423346+0000","last_peered":"2024-09-13T14:46:24.423346+0000","last_clean":"2024-09-13T14:46:24.423346+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:46:24.423346+0000","last_undegraded":"2024-09-13T14:46:24.423346+0000","last_fullsized":"2024-09-13T14:46:24.423346+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":613,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.436023+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:46:24.436023+0000","last_peered":"2024-09-13T14:46:24.436023+0000","last_clean":"2024-09-13T14:46:24.436023+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:46:24.436023+0000","last_undegraded":"2024-09-13T14:46:24.436023+0000","last_fullsized":"2024-09-13T14:46:24.436023+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":56,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.340633+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:46:13.340633+0000","last_peered":"2024-09-13T14:46:13.340633+0000","last_clean":"2024-09-13T14:46:13.340633+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:46:13.340633+0000","last_undegraded":"2024-09-13T14:46:13.340633+0000","last_fullsized":"2024-09-13T14:46:13.340633+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":510,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.423624+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:46:24.423624+0000","last_peered":"2024-09-13T14:46:24.423624+0000","last_clean":"2024-09-13T14:46:24.423624+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:46:24.423624+0000","last_undegraded":"2024-09-13T14:46:24.423624+0000","last_fullsized":"2024-09-13T14:46:24.423624+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":591,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337244+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:46:24.337244+0000","last_peered":"2024-09-13T14:46:24.337244+0000","last_clean":"2024-09-13T14:46:24.337244+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:46:24.337244+0000","last_undegraded":"2024-09-13T14:46:24.337244+0000","last_fullsized":"2024-09-13T14:46:24.337244+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317485+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:46:13.317485+0000","last_peered":"2024-09-13T14:46:13.317485+0000","last_clean":"2024-09-13T14:46:13.317485+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:46:13.317485+0000","last_undegraded":"2024-09-13T14:46:13.317485+0000","last_fullsized":"2024-09-13T14:46:13.317485+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.319703+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:46:13.319703+0000","last_peered":"2024-09-13T14:46:13.319703+0000","last_clean":"2024-09-13T14:46:13.319703+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:46:13.319703+0000","last_undegraded":"2024-09-13T14:46:13.319703+0000","last_fullsized":"2024-09-13T14:46:13.319703+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.340552+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:46:13.340552+0000","last_peered":"2024-09-13T14:46:13.340552+0000","last_clean":"2024-09-13T14:46:13.340552+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:46:13.340552+0000","last_undegraded":"2024-09-13T14:46:13.340552+0000","last_fullsized":"2024-09-13T14:46:13.340552+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":289,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.320165+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:46:13.320165+0000","last_peered":"2024-09-13T14:46:13.320165+0000","last_clean":"2024-09-13T14:46:13.320165+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:46:13.320165+0000","last_undegraded":"2024-09-13T14:46:13.320165+0000","last_fullsized":"2024-09-13T14:46:13.320165+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":610,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.435195+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:46:24.435195+0000","last_peered":"2024-09-13T14:46:24.435195+0000","last_clean":"2024-09-13T14:46:24.435195+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:46:24.435195+0000","last_undegraded":"2024-09-13T14:46:24.435195+0000","last_fullsized":"2024-09-13T14:46:24.435195+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":597,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.436245+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:46:24.436245+0000","last_peered":"2024-09-13T14:46:24.436245+0000","last_clean":"2024-09-13T14:46:24.436245+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:46:24.436245+0000","last_undegraded":"2024-09-13T14:46:24.436245+0000","last_fullsized":"2024-09-13T14:46:24.436245+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-09-13T14:46:13.317119+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:46:13.317119+0000","last_peered":"2024-09-13T14:46:13.317119+0000","last_clean":"2024-09-13T14:46:13.317119+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:46:13.317119+0000","last_undegraded":"2024-09-13T14:46:13.317119+0000","last_fullsized":"2024-09-13T14:46:13.317119+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":601,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337107+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:46:24.337107+0000","last_peered":"2024-09-13T14:46:24.337107+0000","last_clean":"2024-09-13T14:46:24.337107+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:46:24.337107+0000","last_undegraded":"2024-09-13T14:46:24.337107+0000","last_fullsized":"2024-09-13T14:46:24.337107+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3306356760,"num_objects":50452,"num_object_clones":0,"num_object_copies":151356,"num_objects_missing_on_primary":2122,"num_objects_missing":2122,"num_objects_degraded":35332,"num_objects_misplaced":3076,"num_objects_unfound":0,"num_objects_dirty":50452,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":132226,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24901,"num_bytes_recovered":1571487744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9296097280,"data_stored":9296085064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122726,"ondisk_log_size":122726,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397924,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1915612,"kb_used_data":1023764,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91829540,"statfs":{"total":95995035648,"available":94033448960,"internally_reserved":0,"allocated":1048334336,"data_stored":1046432151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594152,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1954376,"kb_used_data":1368640,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91790776,"statfs":{"total":95995035648,"available":93993754624,"internally_reserved":0,"allocated":1401487360,"data_stored":1399604929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665606,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589128,"kb_used_data":252416,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93156024,"statfs":{"total":95995035648,"available":95391768576,"internally_reserved":0,"allocated":258473984,"data_stored":256587884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986607,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648212,"kb_used_data":752524,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92096940,"statfs":{"total":95995035648,"available":94307266560,"internally_reserved":0,"allocated":770584576,"data_stored":768690489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1,1,0,1,3,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697167,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337668,"kb_used_data":1591996,"kb_used_omap":4,"kb_used_meta":745659,"kb_avail":91407484,"statfs":{"total":95995035648,"available":93601263616,"internally_reserved":0,"allocated":1630203904,"data_stored":1628321957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":763554891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860694,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2119884,"kb_used_data":1506948,"kb_used_omap":8,"kb_used_meta":612919,"kb_avail":91625268,"statfs":{"total":95995035648,"available":93824274432,"internally_reserved":0,"allocated":1543114752,"data_stored":1541225151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":627629900},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020365,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733892,"kb_used_data":1430328,"kb_used_omap":9,"kb_used_meta":303542,"kb_avail":92011260,"statfs":{"total":95995035648,"available":94219530240,"internally_reserved":0,"allocated":1464655872,"data_stored":1462777398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":310827845},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187756,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1829440,"kb_used_data":1214776,"kb_used_omap":7,"kb_used_meta":614648,"kb_avail":91915712,"statfs":{"total":95995035648,"available":94121689088,"internally_reserved":0,"allocated":1243930624,"data_stored":1242052416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":629399883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1234173952,"data_stored":1234173952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1455030272,"data_stored":1455030272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1537544192,"data_stored":1537540120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1621426176,"data_stored":1621426176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":765591552,"data_stored":765591552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":252448768,"data_stored":252444696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1393229824,"data_stored":1393229824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1036652544,"data_stored":1036648472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:32.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:46:32.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:32.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:32.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:32.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:32.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:32.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:32.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:32.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:32.443 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:32.445 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:32.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:32.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:32.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:32.574 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:32 smithi017 ceph-mon[103001]: pgmap v1129: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 35283/150093 objects degraded (23.507%); 3026/150093 objects misplaced (2.016%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:32.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:32 smithi017 ceph-mon[107556]: pgmap v1129: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 35283/150093 objects degraded (23.507%); 3026/150093 objects misplaced (2.016%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:32.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:32.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:32 smithi189 ceph-mon[90992]: pgmap v1129: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 35283/150093 objects degraded (23.507%); 3026/150093 objects misplaced (2.016%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:32.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:32.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:32.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:32.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:33.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:33.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:33.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:33.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:33.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:33.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:33.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:33.390 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:33.390 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:33.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:33.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:33.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:33.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:33.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:33.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:33.773 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:33.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:33.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:33.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:33.954 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:34.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:34.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:34.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:34.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:34.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[103001]: pgmap v1130: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 35177/149469 objects degraded (23.535%); 3026/149469 objects misplaced (2.025%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:46:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35177/149469 objects degraded (23.535%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[107556]: pgmap v1130: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 35177/149469 objects degraded (23.535%); 3026/149469 objects misplaced (2.025%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:46:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35177/149469 objects degraded (23.535%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:34.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:34.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:34 smithi189 ceph-mon[90992]: pgmap v1130: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 35177/149469 objects degraded (23.535%); 3026/149469 objects misplaced (2.025%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:46:34.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:34 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:34.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35177/149469 objects degraded (23.535%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:34.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:34.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:46:34.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:34.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:34.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:34.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:35.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:35.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:35.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:35.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:35.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:35.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:35.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:35.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:35.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:35.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:35.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:35.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:35.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:35.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:36.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:36.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:36.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:36.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:36.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:36.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:36.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:36.368 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[103001]: pgmap v1131: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 35128/148137 objects degraded (23.713%); 2977/148137 objects misplaced (2.010%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3451613400' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[107556]: pgmap v1131: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 35128/148137 objects degraded (23.713%); 2977/148137 objects misplaced (2.010%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3451613400' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:36 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:36.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:36.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.695 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:36 smithi189 ceph-mon[90992]: pgmap v1131: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 35128/148137 objects degraded (23.713%); 2977/148137 objects misplaced (2.010%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:46:36.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3451613400' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:36 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:36.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:36.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:37.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:37.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:37.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:37.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:37.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:37.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:37.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:37.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:37.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:37 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:37 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:37 smithi189 ceph-mon[90992]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:46:37.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:37.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:37] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[107556]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:37.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:37.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:37 smithi017 ceph-mon[103001]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:46:37.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:37.699 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:37.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:37.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:37.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:37.930 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:37.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:37.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:38.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:38.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:38.142 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:38.142 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35044/147474 objects degraded (23.763%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1595,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":359,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":49158,"data_bytes":3224406710,"bytes_used":14261391360,"bytes_avail":753698893824,"bytes_total":767960285184,"degraded_objects":35044,"degraded_total":147474,"degraded_ratio":0.23762832770522258,"misplaced_objects":2977,"misplaced_total":147474,"misplaced_ratio":0.020186609165005354,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1271066,"recovering_keys_per_sec":0,"num_objects_recovered":194,"num_bytes_recovered":12713984,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":162},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (7m)\n [==========================..] (remaining: 33s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:46:38.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:38.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:38.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:38 smithi017 ceph-mon[103001]: pgmap v1133: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 35044/147474 objects degraded (23.763%); 2977/147474 objects misplaced (2.019%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/814051177' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:38 smithi017 ceph-mon[107556]: pgmap v1133: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 35044/147474 objects degraded (23.763%); 2977/147474 objects misplaced (2.019%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/814051177' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:38.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.688 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.709 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:38 smithi189 ceph-mon[90992]: pgmap v1133: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 35044/147474 objects degraded (23.763%); 2977/147474 objects misplaced (2.019%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/814051177' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:46:38.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:39.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.084 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.136 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:46:39.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:39.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:39.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:39.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:46:39.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:39.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:39.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:39.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 35044/147474 objects degraded (23.763%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 35044/147474 objects degraded (23.763%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:39.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:39.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:39 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 35044/147474 objects degraded (23.763%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:40.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:40.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:40.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:40.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:40.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:40.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:40.486 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:40.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:40.548 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:46:40.548 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:40.574 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:46:40.575 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:40.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:40.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:46:40.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:40.608 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:46:40.608 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:40.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:40.623 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:46:40.623 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:40.639 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:46:40.639 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:40.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:46:40.656 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:40.677 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:46:40.677 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:40.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:40.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:40.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:40.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:40.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:40 smithi017 ceph-mon[107556]: pgmap v1134: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 35044/146664 objects degraded (23.894%); 2977/146664 objects misplaced (2.030%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1965695995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:40 smithi017 ceph-mon[103001]: pgmap v1134: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 35044/146664 objects degraded (23.894%); 2977/146664 objects misplaced (2.030%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1965695995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:40.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:40 smithi189 ceph-mon[90992]: pgmap v1134: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 35044/146664 objects degraded (23.894%); 2977/146664 objects misplaced (2.030%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1965695995' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:41.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:41.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:41.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:41.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:41.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:41.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:41.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:41.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:41.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:41.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:41.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:41.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:41.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:41.864 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:41.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:41.987 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:42.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:42.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:42.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:42.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:42.549 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:42.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:42.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:42.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:42.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[107556]: pgmap v1135: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34993/145377 objects degraded (24.071%); 2926/145377 objects misplaced (2.013%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[103001]: pgmap v1135: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34993/145377 objects degraded (24.071%); 2926/145377 objects misplaced (2.013%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:42.777 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:42.783 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:42.785 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:46:42.785 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:46:42.785 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:46:42.785 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:46:42.785 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:46:42.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:42.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:42.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:42 smithi189 ceph-mon[90992]: pgmap v1135: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34993/145377 objects degraded (24.071%); 2926/145377 objects misplaced (2.013%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:43.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:43.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:43.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:43.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:43.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:43.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:43.587 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:43.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34993/145377 objects degraded (24.071%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34993/145377 objects degraded (24.071%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:43.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:43.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:43.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:43.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:43.987 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34993/145377 objects degraded (24.071%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:44.089 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:44.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:44.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:44.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:44.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:44.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:44.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:44.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:44.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:44.593 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:44.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:44.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:44.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:44.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:44.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:44 smithi017 ceph-mon[103001]: pgmap v1136: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 34903/144825 objects degraded (24.100%); 2926/144825 objects misplaced (2.020%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:44 smithi017 ceph-mon[107556]: pgmap v1136: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 34903/144825 objects degraded (24.100%); 2926/144825 objects misplaced (2.020%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:44.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:44.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:44 smithi189 ceph-mon[90992]: pgmap v1136: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 34903/144825 objects degraded (24.100%); 2926/144825 objects misplaced (2.020%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:46:45.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:45.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:45.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:46:45.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:45.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:45.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:45.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:45.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:45.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:45.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:45.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:45.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:45.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:45.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:45.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:45.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:45.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:46.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:46.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:46.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:46.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:46.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:46.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:46.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:46.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:46.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:46.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:46.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:46.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:46.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:46.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:46.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:46 smithi017 ceph-mon[103001]: pgmap v1137: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 34850/143667 objects degraded (24.257%); 2872/143667 objects misplaced (1.999%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:46.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:46 smithi017 ceph-mon[107556]: pgmap v1137: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 34850/143667 objects degraded (24.257%); 2872/143667 objects misplaced (1.999%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:46.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:46.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:46.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:46.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:46 smithi189 ceph-mon[90992]: pgmap v1137: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 34850/143667 objects degraded (24.257%); 2872/143667 objects misplaced (1.999%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:46:47.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:47.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:47.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:47.211 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:47.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:47.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:47.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:47.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:47.464 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:47] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.51.0" 2024-09-13T14:46:47.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:47.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:47.586 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:47.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:47.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:47.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:47.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:47.802 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:46:47.803 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:46:47.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:47.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:47.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:48.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:48.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:48.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:48.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:48.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:48.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:48.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:48.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:48.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:48.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:48.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:48.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:48.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:48.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:48.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:48 smithi017 ceph-mon[103001]: pgmap v1138: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 34746/142848 objects degraded (24.324%); 2872/142848 objects misplaced (2.011%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:46:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:48 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34746/142848 objects degraded (24.324%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:48 smithi017 ceph-mon[107556]: pgmap v1138: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 34746/142848 objects degraded (24.324%); 2872/142848 objects misplaced (2.011%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:46:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:48 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34746/142848 objects degraded (24.324%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:49.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:48 smithi189 ceph-mon[90992]: pgmap v1138: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 34746/142848 objects degraded (24.324%); 2872/142848 objects misplaced (2.011%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:46:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:48 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34746/142848 objects degraded (24.324%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:49.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:49.127 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:49.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:49.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:49.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:49.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:49.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:49.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:49.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:49.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:49.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:49.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:49.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:49.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:49.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:49.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:49 smithi017 ceph-mon[107556]: from='client.46409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:49.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:49 smithi017 ceph-mon[103001]: from='client.46409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:49.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:49.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:49.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:49 smithi189 ceph-mon[90992]: from='client.46409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:50.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:50.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:50.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:50.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:50.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:50.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:50.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:50.396 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:50.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:46:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:46:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:50.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:50.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:50.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:50.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:50.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:50 smithi017 ceph-mon[103001]: pgmap v1139: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34746/141813 objects degraded (24.501%); 2872/141813 objects misplaced (2.025%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:46:50.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:50 smithi017 ceph-mon[107556]: pgmap v1139: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34746/141813 objects degraded (24.501%); 2872/141813 objects misplaced (2.025%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:82842 8.e 8:78b86351:::benchmark_data_smithi017_253885_object82841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:46:50.837 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 374.95283550800002, 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.122549702, 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82842 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881957+0000", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.881958+0000", 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.33e-06 2024-09-13T14:46:50.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882008+0000", 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0303999999999998e-05 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882020+0000", 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1847e-05 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.882027+0000", 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0389999999999999e-06 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984746+0000", 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10271846599999999 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.984909+0000", 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016386699999999999 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.985058+0000", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000148479 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.991823+0000", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067648980000000001 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996312+0000", 2024-09-13T14:46:50.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044889270000000002 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996354+0000", 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1857999999999998e-05 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:35.996415+0000", 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1398999999999999e-05 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.004506+0000", 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080912880000000003 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(8.2 314/311 [PushReplyOp(8:4757a975:::benchmark_data_smithi017_253885_object3344:head)])", 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 374.63172287700002, 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14042681400000001, 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.844 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203069+0000", 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203070+0000", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9399999999999993e-07 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203077+0000", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6320000000000002e-06 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203093+0000", 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6152000000000001e-05 2024-09-13T14:46:50.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.203102+0000", 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999998e-06 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.334767+0000", 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13166481199999999 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:36.343496+0000", 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087293370000000002 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88690 8.e 8:774d5387:::benchmark_data_smithi017_253885_object88689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 368.35703595299998, 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13302814199999999, 2024-09-13T14:46:50.848 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88690 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477756+0000", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477757+0000", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1900000000000001e-07 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477779+0000", 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2248000000000001e-05 2024-09-13T14:46:50.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477791+0000", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1515000000000001e-05 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.477799+0000", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8800000000000008e-06 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590385+0000", 2024-09-13T14:46:50.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112586266 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590547+0000", 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162399 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.590706+0000", 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015852800000000001 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607019+0000", 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016312884999999999 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607063+0000", 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4583000000000003e-05 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607112+0000", 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8779000000000002e-05 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607171+0000", 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8609000000000001e-05 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610784+0000", 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003613631 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.854 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:88698 8.e 8:7f48e2e9:::benchmark_data_smithi017_253885_object88697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 368.34539567799999, 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.125047359, 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88698 2024-09-13T14:46:50.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489396+0000", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489398+0000", 2024-09-13T14:46:50.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5430000000000001e-06 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489430+0000", 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1897000000000003e-05 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.489443+0000", 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2904999999999999e-05 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.571286+0000", 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081843369999999999 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.594934+0000", 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023647714 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595081+0000", 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014731100000000001 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.595237+0000", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015634899999999999 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.607076+0000", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011838987 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.610972+0000", 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038959049999999999 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611023+0000", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0523999999999999e-05 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.611085+0000", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2390000000000004e-05 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:42.614444+0000", 2024-09-13T14:46:50.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033584639999999998 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89686 8.2 8.4ba11582 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 367.11108609399997, 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97132002200000001, 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89686 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723706+0000", 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723708+0000", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.531e-06 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723735+0000", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.741e-05 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723752+0000", 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7365000000000001e-05 2024-09-13T14:46:50.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723760+0000", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1240000000000005e-06 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723831+0000", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0325000000000002e-05 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.723842+0000", 2024-09-13T14:46:50.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1249e-05 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695013+0000", 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97117079500000003 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.695026+0000", 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3223e-05 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.865 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.866 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:95761 8.1 e327/303 8:8304b12a:::benchmark_data_smithi017_253885_object4420:head v 327'5803, mlcod=327'5801)", 2024-09-13T14:46:50.866 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 315.084194217, 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15775900900000001, 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750598+0000", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750599+0000", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2399999999999997e-07 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750605+0000", 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9750000000000004e-06 2024-09-13T14:46:50.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.750613+0000", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2930000000000001e-06 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901650+0000", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15103724800000001 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901711+0000", 2024-09-13T14:46:50.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1179999999999994e-05 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901732+0000", 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0166999999999999e-05 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908314+0000", 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065826890000000001 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908357+0000", 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2633000000000003e-05 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:95762 8.c 8:352993b8:::benchmark_data_smithi017_253885_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 315.08053519499998, 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.154645952, 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.871 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95762 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754257+0000", 2024-09-13T14:46:50.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754258+0000", 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5460000000000001e-06 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.754265+0000", 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2899999999999999e-06 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901433+0000", 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.147168615 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901452+0000", 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8652999999999999e-05 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901526+0000", 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3640999999999995e-05 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901644+0000", 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118359 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.901746+0000", 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010203400000000001 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.907274+0000", 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005528112 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908552+0000", 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012774419999999999 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908783+0000", 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023154999999999999 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908846+0000", 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3323000000000003e-05 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:41:35.908903+0000", 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6387000000000001e-05 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104233 8.e 8:7e2470cd:::benchmark_data_smithi017_253885_object12892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 242.01061215300001, 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83912028100000002, 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104233 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824180+0000", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824181+0000", 2024-09-13T14:46:50.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2619999999999999e-06 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824186+0000", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9359999999999998e-06 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824199+0000", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2951e-05 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824205+0000", 2024-09-13T14:46:50.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5690000000000004e-06 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824257+0000", 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2454000000000003e-05 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:48.824265+0000", 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0660000000000004e-06 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656221+0000", 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83195581900000004 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656329+0000", 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010786200000000001 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.656420+0000", 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0880999999999999e-05 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663066+0000", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066462250000000004 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663119+0000", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2852999999999997e-05 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663181+0000", 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2327000000000001e-05 2024-09-13T14:46:50.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663246+0000", 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5300000000000002e-05 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663300+0000", 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3776000000000001e-05 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104242 8.e 8:7779217b:::benchmark_data_smithi017_253885_object12901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 241.791694315, 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.62049879299999999, 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104242 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043098+0000", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043100+0000", 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7519999999999999e-06 2024-09-13T14:46:50.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043107+0000", 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6920000000000002e-06 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043127+0000", 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9409e-05 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043134+0000", 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4070000000000004e-06 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043204+0000", 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0451999999999996e-05 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.043215+0000", 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0719e-05 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659704+0000", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.61648930499999999 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659805+0000", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010100200000000001 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.659923+0000", 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011727600000000001 2024-09-13T14:46:50.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663438+0000", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035150530000000002 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663495+0000", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7058999999999997e-05 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663510+0000", 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5038999999999999e-05 2024-09-13T14:46:50.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663557+0000", 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6634000000000001e-05 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663597+0000", 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9993999999999997e-05 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.891 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104296 8.e 8:77431237:::benchmark_data_smithi017_253885_object12955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 241.382529468, 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.220321039, 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104296 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452263+0000", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452264+0000", 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.677e-06 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452271+0000", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6869999999999997e-06 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452289+0000", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7584e-05 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452296+0000", 2024-09-13T14:46:50.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4390000000000003e-06 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452356+0000", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0322999999999998e-05 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.452365+0000", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3410000000000003e-06 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663647+0000", 2024-09-13T14:46:50.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21128269 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.663780+0000", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133106 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.664086+0000", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030593599999999998 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672353+0000", 2024-09-13T14:46:50.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008266509 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672411+0000", 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8459999999999999e-05 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672453+0000", 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1813000000000001e-05 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672526+0000", 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2769999999999996e-05 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672584+0000", 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7704000000000002e-05 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:104299 8.e 8:7d6c2a47:::benchmark_data_smithi017_253885_object12958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2024-09-13T14:46:50.898 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 241.37102736599999, 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.208997076, 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 104299 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463765+0000", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463766+0000", 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1620000000000001e-06 2024-09-13T14:46:50.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463771+0000", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.865e-06 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463782+0000", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.128e-05 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463787+0000", 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4020000000000001e-06 2024-09-13T14:46:50.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463855+0000", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7737999999999994e-05 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.463868+0000", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2928999999999999e-05 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667607+0000", 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.203738481 2024-09-13T14:46:50.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667720+0000", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113057 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.667828+0000", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108084 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672399+0000", 2024-09-13T14:46:50.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045716810000000002 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672597+0000", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197944 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672659+0000", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1593999999999993e-05 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672724+0000", 2024-09-13T14:46:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5009e-05 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:42:49.672762+0000", 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7849999999999998e-05 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119344 8.e 8:7a29b4dc:::benchmark_data_smithi017_253885_object28003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:51.929905+0000", 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 118.904886847, 2024-09-13T14:46:50.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011586111, 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119344 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929905+0000", 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929905+0000", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929907+0000", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.751e-06 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929914+0000", 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2930000000000003e-06 2024-09-13T14:46:50.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929937+0000", 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2986e-05 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.929947+0000", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5440000000000004e-06 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.930013+0000", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5926999999999994e-05 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.930122+0000", 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109348 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.930220+0000", 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7572999999999996e-05 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.934708+0000", 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004488667 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.934816+0000", 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010796199999999999 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.941368+0000", 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065514049999999997 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.941442+0000", 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4177000000000005e-05 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.941491+0000", 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9478000000000003e-05 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:119362 8.c 8:341d3284:::benchmark_data_smithi017_253885_object28021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:44:51.962136+0000", 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 118.872655779, 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095688140000000001, 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.913 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 119362 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962136+0000", 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962136+0000", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962138+0000", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.311e-06 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962144+0000", 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9710000000000001e-06 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962162+0000", 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8002000000000001e-05 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962170+0000", 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.303e-06 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962240+0000", 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9986000000000004e-05 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962359+0000", 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118861 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.962454+0000", 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5717000000000005e-05 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.966589+0000", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041346960000000002 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.971107+0000", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045175119999999996 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.971603+0000", 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049651999999999997 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.971659+0000", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5387999999999999e-05 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:44:51.971705+0000", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6547e-05 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:121980 8.6 8:6567ce7d:::benchmark_data_smithi017_253885_object30639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e351)", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:14.293143+0000", 2024-09-13T14:46:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 96.541649364999998, 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098341179999999993, 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 121980 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293143+0000", 2024-09-13T14:46:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293143+0000", 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293144+0000", 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2270000000000001e-06 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293149+0000", 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3739999999999997e-06 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293165+0000", 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5642e-05 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.293174+0000", 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7630000000000006e-06 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.294297+0000", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011228049999999999 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.294432+0000", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135835 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.294567+0000", 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134647 2024-09-13T14:46:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.301901+0000", 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073340039999999999 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.302814+0000", 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00091286000000000004 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.302852+0000", 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8572999999999998e-05 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.302923+0000", 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0320000000000004e-05 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:14.302977+0000", 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4067999999999998e-05 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:123763 8.c 8:34bafb00:::benchmark_data_smithi017_253885_object32422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:27.673782+0000", 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 83.161010544999996, 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010145632999999999, 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123763 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673782+0000", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673782+0000", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673783+0000", 2024-09-13T14:46:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9e-06 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673791+0000", 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3799999999999996e-06 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673809+0000", 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8470000000000001e-05 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673820+0000", 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0477e-05 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.673905+0000", 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5372999999999997e-05 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.674033+0000", 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012825 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.674142+0000", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108826 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.683430+0000", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0092873269999999997 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.683471+0000", 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1174000000000003e-05 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.683778+0000", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030757499999999998 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.683857+0000", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9072999999999993e-05 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:27.683927+0000", 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9808000000000007e-05 2024-09-13T14:46:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:127576 8.e 8:7d734908:::benchmark_data_smithi017_253885_object36235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.330505+0000", 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 55.504287494000003, 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.059659197999999997, 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 127576 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330505+0000", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330505+0000", 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330506+0000", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2810000000000001e-06 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330511+0000", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2039999999999996e-06 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330526+0000", 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4919000000000001e-05 2024-09-13T14:46:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330534+0000", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8520000000000004e-06 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330582+0000", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7938000000000001e-05 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330662+0000", 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0255999999999999e-05 2024-09-13T14:46:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330729+0000", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6926000000000003e-05 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.334065+0000", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033363030000000001 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.334084+0000", 2024-09-13T14:46:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9215000000000002e-05 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.390097+0000", 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.056012882 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.390134+0000", 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7100000000000001e-05 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.390164+0000", 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9322000000000001e-05 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:128797 8.e 8:7ac4885a:::benchmark_data_smithi017_253885_object37456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:46:02.857531+0000", 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 47.977261468999998, 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096310149999999997, 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 128797 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857531+0000", 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857531+0000", 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857531+0000", 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8800000000000002e-07 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857534+0000", 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.021e-06 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857543+0000", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6170000000000007e-06 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.857548+0000", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6670000000000001e-06 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.862563+0000", 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050143389999999996 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.862656+0000", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3590000000000003e-05 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.862732+0000", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5667000000000006e-05 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.866853+0000", 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041208590000000002 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.866907+0000", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4114999999999999e-05 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.867067+0000", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015951 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.867121+0000", 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4647e-05 2024-09-13T14:46:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:02.867162+0000", 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0395000000000002e-05 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:131859 8.c 8:318bb43b:::benchmark_data_smithi017_253885_object40518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:46:24.465264+0000", 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.369528418000002, 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012291015000000001, 2024-09-13T14:46:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 131859 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.465264+0000", 2024-09-13T14:46:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.465264+0000", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.465265+0000", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.716e-06 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.465272+0000", 2024-09-13T14:46:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9659999999999999e-06 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.466960+0000", 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016873750000000001 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.466973+0000", 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3125000000000001e-05 2024-09-13T14:46:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.467046+0000", 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2875999999999998e-05 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.467173+0000", 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012749300000000001 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.467286+0000", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011273900000000001 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.474582+0000", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072958520000000002 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.477454+0000", 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028724050000000002 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.477489+0000", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4406999999999998e-05 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.477522+0000", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3630999999999997e-05 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:24.477555+0000", 2024-09-13T14:46:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.243e-05 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:132251 8.e 8:7cc01e35:::benchmark_data_smithi017_253885_object40910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:46:26.988915+0000", 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 23.845876731000001, 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013098820000000001, 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 132251 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988915+0000", 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988915+0000", 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988918+0000", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3130000000000001e-06 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988926+0000", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7059999999999993e-06 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988945+0000", 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8315e-05 2024-09-13T14:46:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.988955+0000", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8789999999999998e-06 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.989022+0000", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7476000000000002e-05 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.989135+0000", 2024-09-13T14:46:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113429 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.989236+0000", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100927 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.992888+0000", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036512300000000001 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:26.992975+0000", 2024-09-13T14:46:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6923999999999996e-05 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:27.001948+0000", 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089736139999999995 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:27.001984+0000", 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5428e-05 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:27.002014+0000", 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0579e-05 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:136209 8.e 8:7e14df1a:::benchmark_data_smithi017_253885_object44868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:46:50.725288+0000", 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.109503793, 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096862560000000007, 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:46:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 136209 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725288+0000", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725288+0000", 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725291+0000", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3429999999999998e-06 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725298+0000", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6389999999999992e-06 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725319+0000", 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0851000000000001e-05 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725329+0000", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.499e-06 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725401+0000", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2268000000000007e-05 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725548+0000", 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014694600000000001 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.725677+0000", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012883299999999999 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.734834+0000", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091575179999999999 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.734885+0000", 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0850999999999998e-05 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.734896+0000", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1462e-05 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.734931+0000", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4847e-05 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:46:50.734975+0000", 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3198999999999998e-05 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.962 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:46:50.963 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:46:50.963 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:46:50.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:50 smithi189 ceph-mon[90992]: pgmap v1139: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 34746/141813 objects degraded (24.501%); 2872/141813 objects misplaced (2.025%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:46:51.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:51.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:51.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:51.234 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:51.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:51.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:51.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:51.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:51.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:51.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:51.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:51.702 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:46:51.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:51.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:51.834 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:51.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2100486464' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:51 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2100486464' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:51 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:51.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:51.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2100486464' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:51 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:52.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:52.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:52.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:52.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:52.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:52.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:52.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:52.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:52.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:52.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:52.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:52.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:52.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:52.861 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:52.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[107556]: pgmap v1140: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 34690/140241 objects degraded (24.736%); 2819/140241 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[107556]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:46:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[103001]: pgmap v1140: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 34690/140241 objects degraded (24.736%); 2819/140241 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:52 smithi017 ceph-mon[103001]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:46:52.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:52.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:52.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:52 smithi189 ceph-mon[90992]: pgmap v1140: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 34690/140241 objects degraded (24.736%); 2819/140241 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:52 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:52 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:52 smithi189 ceph-mon[90992]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:46:53.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:53.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:53.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.230 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:53.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:53.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:46:53.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:53.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:53.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:53.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:53.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:53.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:53.817 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1138,"stamp":"2024-09-13T14:46:47.246805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3123350198,"num_objects":47616,"num_object_clones":0,"num_object_copies":142848,"num_objects_missing_on_primary":1920,"num_objects_missing":1920,"num_objects_degraded":34746,"num_objects_misplaced":2872,"num_objects_unfound":0,"num_objects_dirty":47616,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":136037,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25354,"num_bytes_recovered":1597606045,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125522,"ondisk_log_size":125522,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13703760,"kb_used_data":8696080,"kb_used_omap":44,"kb_used_meta":5007507,"kb_avail":736257456,"statfs":{"total":767960285184,"available":753927634944,"internally_reserved":0,"allocated":8904785920,"data_stored":8889630111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5127687415},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,0,1,0,0,5,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-103743488,"num_objects":-1583,"num_object_clones":0,"num_object_copies":-4749,"num_objects_missing_on_primary":-104,"num_objects_missing":-104,"num_objects_degraded":-382,"num_objects_misplaced":-105,"num_objects_unfound":0,"num_objects_dirty":-1583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1583,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":244,"num_bytes_recovered":15990784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.881753"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":602,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366339+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:46:36.366339+0000","last_peered":"2024-09-13T14:46:36.366339+0000","last_clean":"2024-09-13T14:46:36.366339+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:46:36.366339+0000","last_undegraded":"2024-09-13T14:46:36.366339+0000","last_fullsized":"2024-09-13T14:46:36.366339+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":357,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453462+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:46:24.453462+0000","last_peered":"2024-09-13T14:46:24.453462+0000","last_clean":"2024-09-13T14:46:24.453462+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:46:24.453462+0000","last_undegraded":"2024-09-13T14:46:24.453462+0000","last_fullsized":"2024-09-13T14:46:24.453462+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":286,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340431+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:46:24.340431+0000","last_peered":"2024-09-13T14:46:24.340431+0000","last_clean":"2024-09-13T14:46:24.340431+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:46:24.340431+0000","last_undegraded":"2024-09-13T14:46:24.340431+0000","last_fullsized":"2024-09-13T14:46:24.340431+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":587,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381673+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:46:36.381673+0000","last_peered":"2024-09-13T14:46:36.381673+0000","last_clean":"2024-09-13T14:46:36.381673+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:46:36.381673+0000","last_undegraded":"2024-09-13T14:46:36.381673+0000","last_fullsized":"2024-09-13T14:46:36.381673+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":588,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.386004+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:46:36.386004+0000","last_peered":"2024-09-13T14:46:36.386004+0000","last_clean":"2024-09-13T14:46:36.386004+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:46:36.386004+0000","last_undegraded":"2024-09-13T14:46:36.386004+0000","last_fullsized":"2024-09-13T14:46:36.386004+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":326,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453446+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:46:24.453446+0000","last_peered":"2024-09-13T14:46:24.453446+0000","last_clean":"2024-09-13T14:46:24.453446+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:46:24.453446+0000","last_undegraded":"2024-09-13T14:46:24.453446+0000","last_fullsized":"2024-09-13T14:46:24.453446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":602,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366642+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:46:36.366642+0000","last_peered":"2024-09-13T14:46:36.366642+0000","last_clean":"2024-09-13T14:46:36.366642+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:46:36.366642+0000","last_undegraded":"2024-09-13T14:46:36.366642+0000","last_fullsized":"2024-09-13T14:46:36.366642+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":348,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337242+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:46:24.337242+0000","last_peered":"2024-09-13T14:46:24.337242+0000","last_clean":"2024-09-13T14:46:24.337242+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:46:24.337242+0000","last_undegraded":"2024-09-13T14:46:24.337242+0000","last_fullsized":"2024-09-13T14:46:24.337242+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"359'8414","reported_seq":16841,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:47.049733+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:46:47.049733+0000","last_peered":"2024-09-13T14:46:47.049733+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:46:47.049733+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188481536,"num_objects":2876,"num_object_clones":0,"num_object_copies":8628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4667,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2769,"num_bytes_recovered":181469184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1892}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":352,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453618+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:46:24.453618+0000","last_peered":"2024-09-13T14:46:24.453618+0000","last_clean":"2024-09-13T14:46:24.453618+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:46:24.453618+0000","last_undegraded":"2024-09-13T14:46:24.453618+0000","last_fullsized":"2024-09-13T14:46:24.453618+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339496+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:46:24.339496+0000","last_peered":"2024-09-13T14:46:24.339496+0000","last_clean":"2024-09-13T14:46:24.339496+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:46:24.339496+0000","last_undegraded":"2024-09-13T14:46:24.339496+0000","last_fullsized":"2024-09-13T14:46:24.339496+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"359'8467","reported_seq":18023,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:43.063577+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:46:43.063577+0000","last_peered":"2024-09-13T14:46:43.063577+0000","last_clean":"2024-09-13T14:46:43.063577+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:46:43.063577+0000","last_undegraded":"2024-09-13T14:46:43.063577+0000","last_fullsized":"2024-09-13T14:46:43.063577+0000","mapping_epoch":326,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197984256,"num_objects":3021,"num_object_clones":0,"num_object_copies":9063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":370,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453835+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:46:24.453835+0000","last_peered":"2024-09-13T14:46:24.453835+0000","last_clean":"2024-09-13T14:46:24.453835+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:46:24.453835+0000","last_undegraded":"2024-09-13T14:46:24.453835+0000","last_fullsized":"2024-09-13T14:46:24.453835+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347719+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:46:24.347719+0000","last_peered":"2024-09-13T14:46:24.347719+0000","last_clean":"2024-09-13T14:46:24.347719+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:46:24.347719+0000","last_undegraded":"2024-09-13T14:46:24.347719+0000","last_fullsized":"2024-09-13T14:46:24.347719+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"359'8769","reported_seq":8895,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:43.978855+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:46:43.978855+0000","last_peered":"2024-09-13T14:46:43.978855+0000","last_clean":"2024-09-13T14:46:43.978855+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:46:43.978855+0000","last_undegraded":"2024-09-13T14:46:43.978855+0000","last_fullsized":"2024-09-13T14:46:43.978855+0000","mapping_epoch":303,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8669,"log_dups_size":0,"ondisk_log_size":8669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198508544,"num_objects":3029,"num_object_clones":0,"num_object_copies":9087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8769,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":324,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.455939+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:46:24.455939+0000","last_peered":"2024-09-13T14:46:24.455939+0000","last_clean":"2024-09-13T14:46:24.455939+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:46:24.455939+0000","last_undegraded":"2024-09-13T14:46:24.455939+0000","last_fullsized":"2024-09-13T14:46:24.455939+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":339,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337453+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:46:24.337453+0000","last_peered":"2024-09-13T14:46:24.337453+0000","last_clean":"2024-09-13T14:46:24.337453+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:46:24.337453+0000","last_undegraded":"2024-09-13T14:46:24.337453+0000","last_fullsized":"2024-09-13T14:46:24.337453+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"359'8541","reported_seq":8667,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:45.031937+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:46:45.031937+0000","last_peered":"2024-09-13T14:46:45.031937+0000","last_clean":"2024-09-13T14:46:45.031937+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:46:45.031937+0000","last_undegraded":"2024-09-13T14:46:45.031937+0000","last_fullsized":"2024-09-13T14:46:45.031937+0000","mapping_epoch":303,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196935680,"num_objects":3005,"num_object_clones":0,"num_object_copies":9015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8541,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":287,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340040+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:46:24.340040+0000","last_peered":"2024-09-13T14:46:24.340040+0000","last_clean":"2024-09-13T14:46:24.340040+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:46:24.340040+0000","last_undegraded":"2024-09-13T14:46:24.340040+0000","last_fullsized":"2024-09-13T14:46:24.340040+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":614,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381945+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:46:36.381945+0000","last_peered":"2024-09-13T14:46:36.381945+0000","last_clean":"2024-09-13T14:46:36.381945+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:46:36.381945+0000","last_undegraded":"2024-09-13T14:46:36.381945+0000","last_fullsized":"2024-09-13T14:46:36.381945+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"359'8528","reported_seq":8651,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:45.501093+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:46:45.501093+0000","last_peered":"2024-09-13T14:46:45.501093+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:46:45.501093+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282624,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8528,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2699}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":470,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453009+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:46:24.453009+0000","last_peered":"2024-09-13T14:46:24.453009+0000","last_clean":"2024-09-13T14:46:24.453009+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:46:24.453009+0000","last_undegraded":"2024-09-13T14:46:24.453009+0000","last_fullsized":"2024-09-13T14:46:24.453009+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":612,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366245+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:46:36.366245+0000","last_peered":"2024-09-13T14:46:36.366245+0000","last_clean":"2024-09-13T14:46:36.366245+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:46:36.366245+0000","last_undegraded":"2024-09-13T14:46:36.366245+0000","last_fullsized":"2024-09-13T14:46:36.366245+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347765+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:46:24.347765+0000","last_peered":"2024-09-13T14:46:24.347765+0000","last_clean":"2024-09-13T14:46:24.347765+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:46:24.347765+0000","last_undegraded":"2024-09-13T14:46:24.347765+0000","last_fullsized":"2024-09-13T14:46:24.347765+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"359'8156","reported_seq":8282,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:45.023393+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:46:45.023393+0000","last_peered":"2024-09-13T14:46:45.023393+0000","last_clean":"2024-09-13T14:46:45.023393+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:46:45.023393+0000","last_undegraded":"2024-09-13T14:46:45.023393+0000","last_fullsized":"2024-09-13T14:46:45.023393+0000","mapping_epoch":303,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192544768,"num_objects":2938,"num_object_clones":0,"num_object_copies":8814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8156,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":174,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339510+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:46:24.339510+0000","last_peered":"2024-09-13T14:46:24.339510+0000","last_clean":"2024-09-13T14:46:24.339510+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:46:24.339510+0000","last_undegraded":"2024-09-13T14:46:24.339510+0000","last_fullsized":"2024-09-13T14:46:24.339510+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":360,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337381+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:46:24.337381+0000","last_peered":"2024-09-13T14:46:24.337381+0000","last_clean":"2024-09-13T14:46:24.337381+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:46:24.337381+0000","last_undegraded":"2024-09-13T14:46:24.337381+0000","last_fullsized":"2024-09-13T14:46:24.337381+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":72,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348488+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:46:24.348488+0000","last_peered":"2024-09-13T14:46:24.348488+0000","last_clean":"2024-09-13T14:46:24.348488+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:46:24.348488+0000","last_undegraded":"2024-09-13T14:46:24.348488+0000","last_fullsized":"2024-09-13T14:46:24.348488+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"359'8436","reported_seq":16348,"reported_epoch":359,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:44.930143+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:46:44.930143+0000","last_peered":"2024-09-13T14:46:44.930143+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:46:44.930143+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7336,"log_dups_size":0,"ondisk_log_size":7336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":1920,"num_objects_missing":1920,"num_objects_degraded":2295,"num_objects_misplaced":2872,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8436,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2731,"num_bytes_recovered":178978816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1920},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347570+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:46:24.347570+0000","last_peered":"2024-09-13T14:46:24.347570+0000","last_clean":"2024-09-13T14:46:24.347570+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:46:24.347570+0000","last_undegraded":"2024-09-13T14:46:24.347570+0000","last_fullsized":"2024-09-13T14:46:24.347570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348517+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:46:24.348517+0000","last_peered":"2024-09-13T14:46:24.348517+0000","last_clean":"2024-09-13T14:46:24.348517+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:46:24.348517+0000","last_undegraded":"2024-09-13T14:46:24.348517+0000","last_fullsized":"2024-09-13T14:46:24.348517+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"359'8235","reported_seq":8361,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:45.046837+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:46:45.046837+0000","last_peered":"2024-09-13T14:46:45.046837+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:46:45.046837+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191561728,"num_objects":2923,"num_object_clones":0,"num_object_copies":8769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8235,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2543}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":167,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339762+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:46:24.339762+0000","last_peered":"2024-09-13T14:46:24.339762+0000","last_clean":"2024-09-13T14:46:24.339762+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:46:24.339762+0000","last_undegraded":"2024-09-13T14:46:24.339762+0000","last_fullsized":"2024-09-13T14:46:24.339762+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":604,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372648+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:46:36.372648+0000","last_peered":"2024-09-13T14:46:36.372648+0000","last_clean":"2024-09-13T14:46:36.372648+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:46:36.372648+0000","last_undegraded":"2024-09-13T14:46:36.372648+0000","last_fullsized":"2024-09-13T14:46:36.372648+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":566,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.375005+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:46:36.375005+0000","last_peered":"2024-09-13T14:46:36.375005+0000","last_clean":"2024-09-13T14:46:36.375005+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:46:36.375005+0000","last_undegraded":"2024-09-13T14:46:36.375005+0000","last_fullsized":"2024-09-13T14:46:36.375005+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"359'8245","reported_seq":8316,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:43.031607+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:46:43.031607+0000","last_peered":"2024-09-13T14:46:43.031607+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:46:43.031607+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194969600,"num_objects":2975,"num_object_clones":0,"num_object_copies":8925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2521}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":461,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453263+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:46:24.453263+0000","last_peered":"2024-09-13T14:46:24.453263+0000","last_clean":"2024-09-13T14:46:24.453263+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:46:24.453263+0000","last_undegraded":"2024-09-13T14:46:24.453263+0000","last_fullsized":"2024-09-13T14:46:24.453263+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":883,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340098+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:46:24.340098+0000","last_peered":"2024-09-13T14:46:24.340098+0000","last_clean":"2024-09-13T14:46:24.340098+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:46:24.340098+0000","last_undegraded":"2024-09-13T14:46:24.340098+0000","last_fullsized":"2024-09-13T14:46:24.340098+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":557,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366080+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:46:36.366080+0000","last_peered":"2024-09-13T14:46:36.366080+0000","last_clean":"2024-09-13T14:46:36.366080+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:46:36.366080+0000","last_undegraded":"2024-09-13T14:46:36.366080+0000","last_fullsized":"2024-09-13T14:46:36.366080+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":325,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.338084+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:46:24.338084+0000","last_peered":"2024-09-13T14:46:24.338084+0000","last_clean":"2024-09-13T14:46:24.338084+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:46:24.338084+0000","last_undegraded":"2024-09-13T14:46:24.338084+0000","last_fullsized":"2024-09-13T14:46:24.338084+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"359'8577","reported_seq":8701,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:45.062017+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:46:45.062017+0000","last_peered":"2024-09-13T14:46:45.062017+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:46:45.062017+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7577,"log_dups_size":0,"ondisk_log_size":7577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194707456,"num_objects":2971,"num_object_clones":0,"num_object_copies":8913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8577,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2709}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":411,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.376597+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:46:36.376597+0000","last_peered":"2024-09-13T14:46:36.376597+0000","last_clean":"2024-09-13T14:46:36.376597+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:46:36.376597+0000","last_undegraded":"2024-09-13T14:46:36.376597+0000","last_fullsized":"2024-09-13T14:46:36.376597+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":749,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453184+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:46:24.453184+0000","last_peered":"2024-09-13T14:46:24.453184+0000","last_clean":"2024-09-13T14:46:24.453184+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:46:24.453184+0000","last_undegraded":"2024-09-13T14:46:24.453184+0000","last_fullsized":"2024-09-13T14:46:24.453184+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":607,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381680+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:46:36.381680+0000","last_peered":"2024-09-13T14:46:36.381680+0000","last_clean":"2024-09-13T14:46:36.381680+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:46:36.381680+0000","last_undegraded":"2024-09-13T14:46:36.381680+0000","last_fullsized":"2024-09-13T14:46:36.381680+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":611,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.386102+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:46:36.386102+0000","last_peered":"2024-09-13T14:46:36.386102+0000","last_clean":"2024-09-13T14:46:36.386102+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:46:36.386102+0000","last_undegraded":"2024-09-13T14:46:36.386102+0000","last_fullsized":"2024-09-13T14:46:36.386102+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"359'8561","reported_seq":18228,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:45.525465+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:46:45.525465+0000","last_peered":"2024-09-13T14:46:45.525465+0000","last_clean":"2024-09-13T14:46:45.525465+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:46:45.525465+0000","last_undegraded":"2024-09-13T14:46:45.525465+0000","last_fullsized":"2024-09-13T14:46:45.525465+0000","mapping_epoch":330,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203358208,"num_objects":3103,"num_object_clones":0,"num_object_copies":9309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8561,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":490,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.453217+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:46:24.453217+0000","last_peered":"2024-09-13T14:46:24.453217+0000","last_clean":"2024-09-13T14:46:24.453217+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:46:24.453217+0000","last_undegraded":"2024-09-13T14:46:24.453217+0000","last_fullsized":"2024-09-13T14:46:24.453217+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":575,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366183+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:46:36.366183+0000","last_peered":"2024-09-13T14:46:36.366183+0000","last_clean":"2024-09-13T14:46:36.366183+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:46:36.366183+0000","last_undegraded":"2024-09-13T14:46:36.366183+0000","last_fullsized":"2024-09-13T14:46:36.366183+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":544,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.376674+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:46:36.376674+0000","last_peered":"2024-09-13T14:46:36.376674+0000","last_clean":"2024-09-13T14:46:36.376674+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:46:36.376674+0000","last_undegraded":"2024-09-13T14:46:36.376674+0000","last_fullsized":"2024-09-13T14:46:36.376674+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339834+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:46:24.339834+0000","last_peered":"2024-09-13T14:46:24.339834+0000","last_clean":"2024-09-13T14:46:24.339834+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:46:24.339834+0000","last_undegraded":"2024-09-13T14:46:24.339834+0000","last_fullsized":"2024-09-13T14:46:24.339834+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"359'8251","reported_seq":8376,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:45.393320+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:46:45.393320+0000","last_peered":"2024-09-13T14:46:45.393320+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:46:45.393320+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195231744,"num_objects":2979,"num_object_clones":0,"num_object_copies":8937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8251,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2530}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":167,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339548+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:46:24.339548+0000","last_peered":"2024-09-13T14:46:24.339548+0000","last_clean":"2024-09-13T14:46:24.339548+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:46:24.339548+0000","last_undegraded":"2024-09-13T14:46:24.339548+0000","last_fullsized":"2024-09-13T14:46:24.339548+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":974,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339858+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:46:24.339858+0000","last_peered":"2024-09-13T14:46:24.339858+0000","last_clean":"2024-09-13T14:46:24.339858+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:46:24.339858+0000","last_undegraded":"2024-09-13T14:46:24.339858+0000","last_fullsized":"2024-09-13T14:46:24.339858+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348088+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:46:24.348088+0000","last_peered":"2024-09-13T14:46:24.348088+0000","last_clean":"2024-09-13T14:46:24.348088+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:46:24.348088+0000","last_undegraded":"2024-09-13T14:46:24.348088+0000","last_fullsized":"2024-09-13T14:46:24.348088+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":564,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374260+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:46:36.374260+0000","last_peered":"2024-09-13T14:46:36.374260+0000","last_clean":"2024-09-13T14:46:36.374260+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:46:36.374260+0000","last_undegraded":"2024-09-13T14:46:36.374260+0000","last_fullsized":"2024-09-13T14:46:36.374260+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"359'8414","reported_seq":17820,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:43.967730+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:46:43.967730+0000","last_peered":"2024-09-13T14:46:43.967730+0000","last_clean":"2024-09-13T14:46:43.967730+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:46:43.967730+0000","last_undegraded":"2024-09-13T14:46:43.967730+0000","last_fullsized":"2024-09-13T14:46:43.967730+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193069056,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":358,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337752+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:46:24.337752+0000","last_peered":"2024-09-13T14:46:24.337752+0000","last_clean":"2024-09-13T14:46:24.337752+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:46:24.337752+0000","last_undegraded":"2024-09-13T14:46:24.337752+0000","last_fullsized":"2024-09-13T14:46:24.337752+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340331+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:46:24.340331+0000","last_peered":"2024-09-13T14:46:24.340331+0000","last_clean":"2024-09-13T14:46:24.340331+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:46:24.340331+0000","last_undegraded":"2024-09-13T14:46:24.340331+0000","last_fullsized":"2024-09-13T14:46:24.340331+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1143,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374357+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:46:36.374357+0000","last_peered":"2024-09-13T14:46:36.374357+0000","last_clean":"2024-09-13T14:46:36.374357+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:46:36.374357+0000","last_undegraded":"2024-09-13T14:46:36.374357+0000","last_fullsized":"2024-09-13T14:46:36.374357+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":545,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374721+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:46:36.374721+0000","last_peered":"2024-09-13T14:46:36.374721+0000","last_clean":"2024-09-13T14:46:36.374721+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:46:36.374721+0000","last_undegraded":"2024-09-13T14:46:36.374721+0000","last_fullsized":"2024-09-13T14:46:36.374721+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"359'8290","reported_seq":8416,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:45.536232+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:46:45.536232+0000","last_peered":"2024-09-13T14:46:45.536232+0000","last_clean":"2024-09-13T14:46:45.536232+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:46:45.536232+0000","last_undegraded":"2024-09-13T14:46:45.536232+0000","last_fullsized":"2024-09-13T14:46:45.536232+0000","mapping_epoch":303,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190971904,"num_objects":2914,"num_object_clones":0,"num_object_copies":8742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8290,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":358,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337867+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:46:24.337867+0000","last_peered":"2024-09-13T14:46:24.337867+0000","last_clean":"2024-09-13T14:46:24.337867+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:46:24.337867+0000","last_undegraded":"2024-09-13T14:46:24.337867+0000","last_fullsized":"2024-09-13T14:46:24.337867+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":554,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374247+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:46:36.374247+0000","last_peered":"2024-09-13T14:46:36.374247+0000","last_clean":"2024-09-13T14:46:36.374247+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:46:36.374247+0000","last_undegraded":"2024-09-13T14:46:36.374247+0000","last_fullsized":"2024-09-13T14:46:36.374247+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":753,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.455955+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:46:24.455955+0000","last_peered":"2024-09-13T14:46:24.455955+0000","last_clean":"2024-09-13T14:46:24.455955+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:46:24.455955+0000","last_undegraded":"2024-09-13T14:46:24.455955+0000","last_fullsized":"2024-09-13T14:46:24.455955+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":590,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381895+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:46:36.381895+0000","last_peered":"2024-09-13T14:46:36.381895+0000","last_clean":"2024-09-13T14:46:36.381895+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:46:36.381895+0000","last_undegraded":"2024-09-13T14:46:36.381895+0000","last_fullsized":"2024-09-13T14:46:36.381895+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"359'8672","reported_seq":8798,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:43.860620+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:46:43.860620+0000","last_peered":"2024-09-13T14:46:43.860620+0000","last_clean":"2024-09-13T14:46:43.860620+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:46:43.860620+0000","last_undegraded":"2024-09-13T14:46:43.860620+0000","last_fullsized":"2024-09-13T14:46:43.860620+0000","mapping_epoch":303,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8572,"log_dups_size":0,"ondisk_log_size":8572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197787648,"num_objects":3018,"num_object_clones":0,"num_object_copies":9054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8672,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348412+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:46:24.348412+0000","last_peered":"2024-09-13T14:46:24.348412+0000","last_clean":"2024-09-13T14:46:24.348412+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:46:24.348412+0000","last_undegraded":"2024-09-13T14:46:24.348412+0000","last_fullsized":"2024-09-13T14:46:24.348412+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":590,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366804+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:46:36.366804+0000","last_peered":"2024-09-13T14:46:36.366804+0000","last_clean":"2024-09-13T14:46:36.366804+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:46:36.366804+0000","last_undegraded":"2024-09-13T14:46:36.366804+0000","last_fullsized":"2024-09-13T14:46:36.366804+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":930,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374682+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:46:36.374682+0000","last_peered":"2024-09-13T14:46:36.374682+0000","last_clean":"2024-09-13T14:46:36.374682+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:46:36.374682+0000","last_undegraded":"2024-09-13T14:46:36.374682+0000","last_fullsized":"2024-09-13T14:46:36.374682+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":588,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372538+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:46:36.372538+0000","last_peered":"2024-09-13T14:46:36.372538+0000","last_clean":"2024-09-13T14:46:36.372538+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:46:36.372538+0000","last_undegraded":"2024-09-13T14:46:36.372538+0000","last_fullsized":"2024-09-13T14:46:36.372538+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"359'8534","reported_seq":19904,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:42.499412+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:46:42.499412+0000","last_peered":"2024-09-13T14:46:42.499412+0000","last_clean":"2024-09-13T14:46:42.499412+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:46:42.499412+0000","last_undegraded":"2024-09-13T14:46:42.499412+0000","last_fullsized":"2024-09-13T14:46:42.499412+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8534,"log_dups_size":0,"ondisk_log_size":8534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195231768,"num_objects":2980,"num_object_clones":0,"num_object_copies":8940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2980,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8534,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":441,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366160+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:46:36.366160+0000","last_peered":"2024-09-13T14:46:36.366160+0000","last_clean":"2024-09-13T14:46:36.366160+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:46:36.366160+0000","last_undegraded":"2024-09-13T14:46:36.366160+0000","last_fullsized":"2024-09-13T14:46:36.366160+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":625,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.385962+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:46:36.385962+0000","last_peered":"2024-09-13T14:46:36.385962+0000","last_clean":"2024-09-13T14:46:36.385962+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:46:36.385962+0000","last_undegraded":"2024-09-13T14:46:36.385962+0000","last_fullsized":"2024-09-13T14:46:36.385962+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":777,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337905+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:46:24.337905+0000","last_peered":"2024-09-13T14:46:24.337905+0000","last_clean":"2024-09-13T14:46:24.337905+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:46:24.337905+0000","last_undegraded":"2024-09-13T14:46:24.337905+0000","last_fullsized":"2024-09-13T14:46:24.337905+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":72,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347661+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:46:24.347661+0000","last_peered":"2024-09-13T14:46:24.347661+0000","last_clean":"2024-09-13T14:46:24.347661+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:46:24.347661+0000","last_undegraded":"2024-09-13T14:46:24.347661+0000","last_fullsized":"2024-09-13T14:46:24.347661+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348157+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:46:24.348157+0000","last_peered":"2024-09-13T14:46:24.348157+0000","last_clean":"2024-09-13T14:46:24.348157+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:46:24.348157+0000","last_undegraded":"2024-09-13T14:46:24.348157+0000","last_fullsized":"2024-09-13T14:46:24.348157+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":68,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347715+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:46:24.347715+0000","last_peered":"2024-09-13T14:46:24.347715+0000","last_clean":"2024-09-13T14:46:24.347715+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:46:24.347715+0000","last_undegraded":"2024-09-13T14:46:24.347715+0000","last_fullsized":"2024-09-13T14:46:24.347715+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":542,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374553+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:46:36.374553+0000","last_peered":"2024-09-13T14:46:36.374553+0000","last_clean":"2024-09-13T14:46:36.374553+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:46:36.374553+0000","last_undegraded":"2024-09-13T14:46:36.374553+0000","last_fullsized":"2024-09-13T14:46:36.374553+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":358,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.455861+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:46:24.455861+0000","last_peered":"2024-09-13T14:46:24.455861+0000","last_clean":"2024-09-13T14:46:24.455861+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:46:24.455861+0000","last_undegraded":"2024-09-13T14:46:24.455861+0000","last_fullsized":"2024-09-13T14:46:24.455861+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":324,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.339696+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:46:24.339696+0000","last_peered":"2024-09-13T14:46:24.339696+0000","last_clean":"2024-09-13T14:46:24.339696+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:46:24.339696+0000","last_undegraded":"2024-09-13T14:46:24.339696+0000","last_fullsized":"2024-09-13T14:46:24.339696+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":546,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.365971+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:46:36.365971+0000","last_peered":"2024-09-13T14:46:36.365971+0000","last_clean":"2024-09-13T14:46:36.365971+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:46:36.365971+0000","last_undegraded":"2024-09-13T14:46:36.365971+0000","last_fullsized":"2024-09-13T14:46:36.365971+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":598,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381848+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:46:36.381848+0000","last_peered":"2024-09-13T14:46:36.381848+0000","last_clean":"2024-09-13T14:46:36.381848+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:46:36.381848+0000","last_undegraded":"2024-09-13T14:46:36.381848+0000","last_fullsized":"2024-09-13T14:46:36.381848+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":602,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372486+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:46:36.372486+0000","last_peered":"2024-09-13T14:46:36.372486+0000","last_clean":"2024-09-13T14:46:36.372486+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:46:36.372486+0000","last_undegraded":"2024-09-13T14:46:36.372486+0000","last_fullsized":"2024-09-13T14:46:36.372486+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":356,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.337945+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:46:24.337945+0000","last_peered":"2024-09-13T14:46:24.337945+0000","last_clean":"2024-09-13T14:46:24.337945+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:46:24.337945+0000","last_undegraded":"2024-09-13T14:46:24.337945+0000","last_fullsized":"2024-09-13T14:46:24.337945+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":620,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372608+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:46:36.372608+0000","last_peered":"2024-09-13T14:46:36.372608+0000","last_clean":"2024-09-13T14:46:36.372608+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:46:36.372608+0000","last_undegraded":"2024-09-13T14:46:36.372608+0000","last_fullsized":"2024-09-13T14:46:36.372608+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":590,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.386072+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:46:36.386072+0000","last_peered":"2024-09-13T14:46:36.386072+0000","last_clean":"2024-09-13T14:46:36.386072+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:46:36.386072+0000","last_undegraded":"2024-09-13T14:46:36.386072+0000","last_fullsized":"2024-09-13T14:46:36.386072+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":566,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374145+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:46:36.374145+0000","last_peered":"2024-09-13T14:46:36.374145+0000","last_clean":"2024-09-13T14:46:36.374145+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:46:36.374145+0000","last_undegraded":"2024-09-13T14:46:36.374145+0000","last_fullsized":"2024-09-13T14:46:36.374145+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":616,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366301+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:46:36.366301+0000","last_peered":"2024-09-13T14:46:36.366301+0000","last_clean":"2024-09-13T14:46:36.366301+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:46:36.366301+0000","last_undegraded":"2024-09-13T14:46:36.366301+0000","last_fullsized":"2024-09-13T14:46:36.366301+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.354935+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:46:24.354935+0000","last_peered":"2024-09-13T14:46:24.354935+0000","last_clean":"2024-09-13T14:46:24.354935+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:46:24.354935+0000","last_undegraded":"2024-09-13T14:46:24.354935+0000","last_fullsized":"2024-09-13T14:46:24.354935+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":512,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374473+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:46:36.374473+0000","last_peered":"2024-09-13T14:46:36.374473+0000","last_clean":"2024-09-13T14:46:36.374473+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:46:36.374473+0000","last_undegraded":"2024-09-13T14:46:36.374473+0000","last_fullsized":"2024-09-13T14:46:36.374473+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":594,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381790+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:46:36.381790+0000","last_peered":"2024-09-13T14:46:36.381790+0000","last_clean":"2024-09-13T14:46:36.381790+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:46:36.381790+0000","last_undegraded":"2024-09-13T14:46:36.381790+0000","last_fullsized":"2024-09-13T14:46:36.381790+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.348016+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:46:24.348016+0000","last_peered":"2024-09-13T14:46:24.348016+0000","last_clean":"2024-09-13T14:46:24.348016+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:46:24.348016+0000","last_undegraded":"2024-09-13T14:46:24.348016+0000","last_fullsized":"2024-09-13T14:46:24.348016+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":324,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340131+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:46:24.340131+0000","last_peered":"2024-09-13T14:46:24.340131+0000","last_clean":"2024-09-13T14:46:24.340131+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:46:24.340131+0000","last_undegraded":"2024-09-13T14:46:24.340131+0000","last_fullsized":"2024-09-13T14:46:24.340131+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.354888+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:46:24.354888+0000","last_peered":"2024-09-13T14:46:24.354888+0000","last_clean":"2024-09-13T14:46:24.354888+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:46:24.354888+0000","last_undegraded":"2024-09-13T14:46:24.354888+0000","last_fullsized":"2024-09-13T14:46:24.354888+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":290,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.340441+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:46:24.340441+0000","last_peered":"2024-09-13T14:46:24.340441+0000","last_clean":"2024-09-13T14:46:24.340441+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:46:24.340441+0000","last_undegraded":"2024-09-13T14:46:24.340441+0000","last_fullsized":"2024-09-13T14:46:24.340441+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":612,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366245+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:46:36.366245+0000","last_peered":"2024-09-13T14:46:36.366245+0000","last_clean":"2024-09-13T14:46:36.366245+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:46:36.366245+0000","last_undegraded":"2024-09-13T14:46:36.366245+0000","last_fullsized":"2024-09-13T14:46:36.366245+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":599,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366575+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:46:36.366575+0000","last_peered":"2024-09-13T14:46:36.366575+0000","last_clean":"2024-09-13T14:46:36.366575+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:46:36.366575+0000","last_undegraded":"2024-09-13T14:46:36.366575+0000","last_fullsized":"2024-09-13T14:46:36.366575+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-09-13T14:46:24.347540+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:46:24.347540+0000","last_peered":"2024-09-13T14:46:24.347540+0000","last_clean":"2024-09-13T14:46:24.347540+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:46:24.347540+0000","last_undegraded":"2024-09-13T14:46:24.347540+0000","last_fullsized":"2024-09-13T14:46:24.347540+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":604,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.381915+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:46:36.381915+0000","last_peered":"2024-09-13T14:46:36.381915+0000","last_clean":"2024-09-13T14:46:36.381915+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:46:36.381915+0000","last_undegraded":"2024-09-13T14:46:36.381915+0000","last_fullsized":"2024-09-13T14:46:36.381915+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3118661656,"num_objects":47588,"num_object_clones":0,"num_object_copies":142764,"num_objects_missing_on_primary":1920,"num_objects_missing":1920,"num_objects_degraded":34746,"num_objects_misplaced":2872,"num_objects_unfound":0,"num_objects_dirty":47588,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135090,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25297,"num_bytes_recovered":1597440000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8840097792,"data_stored":8840085576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123390,"ondisk_log_size":123390,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397927,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1871020,"kb_used_data":975012,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":91874132,"statfs":{"total":95995035648,"available":94079111168,"internally_reserved":0,"allocated":998412288,"data_stored":996502256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594155,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1911704,"kb_used_data":1321808,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":91833448,"statfs":{"total":95995035648,"available":94037450752,"internally_reserved":0,"allocated":1353531392,"data_stored":1351641114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665609,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":571672,"kb_used_data":243280,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93173480,"statfs":{"total":95995035648,"available":95409643520,"internally_reserved":0,"allocated":249118720,"data_stored":247224773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986611,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615076,"kb_used_data":715228,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92130076,"statfs":{"total":95995035648,"available":94341197824,"internally_reserved":0,"allocated":732393472,"data_stored":730491538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,1,0,0,5,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697171,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2264276,"kb_used_data":1514444,"kb_used_omap":4,"kb_used_meta":749819,"kb_avail":91480876,"statfs":{"total":95995035648,"available":93676417024,"internally_reserved":0,"allocated":1550790656,"data_stored":1548900862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":767814731},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860698,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2046876,"kb_used_data":1429716,"kb_used_omap":8,"kb_used_meta":617079,"kb_avail":91698276,"statfs":{"total":95995035648,"available":93899034624,"internally_reserved":0,"allocated":1464029184,"data_stored":1462131736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":631889740},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020369,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648592,"kb_used_data":1340872,"kb_used_omap":9,"kb_used_meta":307702,"kb_avail":92096560,"statfs":{"total":95995035648,"available":94306877440,"internally_reserved":0,"allocated":1373052928,"data_stored":1371166607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":315087685},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187759,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1774544,"kb_used_data":1155720,"kb_used_omap":7,"kb_used_meta":618808,"kb_avail":91970608,"statfs":{"total":95995035648,"available":94177902592,"internally_reserved":0,"allocated":1183457280,"data_stored":1181571225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":633659723},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1172570112,"data_stored":1172570112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1363738624,"data_stored":1363738624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1458704384,"data_stored":1458700312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1541996544,"data_stored":1541996544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":726859776,"data_stored":726859776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":243077120,"data_stored":243073048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1346306048,"data_stored":1346306048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":986845184,"data_stored":986841112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:53.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:54.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:53 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34690/140241 objects degraded (24.736%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:54.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:54.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:53 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34690/140241 objects degraded (24.736%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:53 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34690/140241 objects degraded (24.736%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:54.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:54.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:54.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:54.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:54.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:54.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:54.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:54.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:54.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:54.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:54.745 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:54.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:54.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:46:54.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:54 smithi017 ceph-mon[107556]: pgmap v1142: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 34584/139515 objects degraded (24.789%); 2819/139515 objects misplaced (2.021%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:54.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:54 smithi017 ceph-mon[103001]: pgmap v1142: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 34584/139515 objects degraded (24.789%); 2819/139515 objects misplaced (2.021%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:54.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:54.868 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:54.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:54.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:54 smithi189 ceph-mon[90992]: pgmap v1142: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 34584/139515 objects degraded (24.789%); 2819/139515 objects misplaced (2.021%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:55.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:55.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:55.204 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:46:55.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:55.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:46:55.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:55.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.257 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:46:55.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:55.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:46:55.272 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:55.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:46:55.288 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:55.303 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:46:55.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:55.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:46:55.319 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:55.338 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:46:55.338 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:55.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:55.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:55.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:55.596 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:55.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.725 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:55.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:55.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/36697289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:56.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/36697289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/36697289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:56.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:56.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:56.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:56.474 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:56.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.507 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:56.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:56.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:56.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:56.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:56.988 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:56 smithi017 ceph-mon[103001]: pgmap v1143: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 34529/138201 objects degraded (24.985%); 2763/138201 objects misplaced (1.999%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:56.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:56 smithi017 ceph-mon[107556]: pgmap v1143: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 34529/138201 objects degraded (24.985%); 2763/138201 objects misplaced (1.999%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:56 smithi189 ceph-mon[90992]: pgmap v1143: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 34529/138201 objects degraded (24.985%); 2763/138201 objects misplaced (1.999%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:57.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:57.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:57.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:57.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:57.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:57.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:46:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:46:57] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-13T14:46:57.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:57.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:58.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:58.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.143 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:58.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.268 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:58.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[107556]: pgmap v1144: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 34425/137415 objects degraded (25.052%); 2763/137415 objects misplaced (2.011%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[103001]: pgmap v1144: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 34425/137415 objects degraded (25.052%); 2763/137415 objects misplaced (2.011%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:58.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:58.492 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:58.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:58 smithi189 ceph-mon[90992]: pgmap v1144: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 34425/137415 objects degraded (25.052%); 2763/137415 objects misplaced (2.011%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:46:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:58.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:58.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:58.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:58.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:58.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:59.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.152 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:59.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34425/137415 objects degraded (25.052%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:59.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:46:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34425/137415 objects degraded (25.052%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:59.224 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:59.224 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:59.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:59.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:59.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.505 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:59.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:59.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:46:59.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:46:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34425/137415 objects degraded (25.052%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:46:59.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:46:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:59.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:46:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:00.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:00.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:00.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:00.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:00.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:00.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:00.397 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:00.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:00 smithi189 ceph-mon[90992]: pgmap v1145: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 34425/136536 objects degraded (25.213%); 2763/136536 objects misplaced (2.024%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:00.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:00 smithi017 ceph-mon[103001]: pgmap v1145: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 34425/136536 objects degraded (25.213%); 2763/136536 objects misplaced (2.024%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:00 smithi017 ceph-mon[107556]: pgmap v1145: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 34425/136536 objects degraded (25.213%); 2763/136536 objects misplaced (2.024%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:00.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:00.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:00.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:00.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:00.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:00.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:00.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:01.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:47:01.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:01.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:01.253 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:01.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:01.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:01.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:01.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:02.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:02.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:02.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:02.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:02.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:02.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:02.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:02.525 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:02.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:02.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:02 smithi017 ceph-mon[103001]: pgmap v1146: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34370/135258 objects degraded (25.411%); 2705/135258 objects misplaced (2.000%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T14:47:02.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:02 smithi017 ceph-mon[107556]: pgmap v1146: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34370/135258 objects degraded (25.411%); 2705/135258 objects misplaced (2.000%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T14:47:02.659 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:02 smithi189 ceph-mon[90992]: pgmap v1146: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34370/135258 objects degraded (25.411%); 2705/135258 objects misplaced (2.000%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T14:47:02.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:02.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:02.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.910 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:02.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.159 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.261 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:03.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:03.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:03.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:03.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:03.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:03.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:03.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.910 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:03.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.986 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:04.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:04.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:04.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:04.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:04.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:04.182 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:04.183 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:47:04.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:04.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:04.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:04 smithi017 ceph-mon[103001]: pgmap v1147: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 34276/134739 objects degraded (25.439%); 2705/134739 objects misplaced (2.008%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34276/134739 objects degraded (25.439%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:04 smithi017 ceph-mon[107556]: pgmap v1147: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 34276/134739 objects degraded (25.439%); 2705/134739 objects misplaced (2.008%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34276/134739 objects degraded (25.439%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:04.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:04.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:04.643 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:04.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:04.754 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:04.754 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:04 smithi189 ceph-mon[90992]: pgmap v1147: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 34276/134739 objects degraded (25.439%); 2705/134739 objects misplaced (2.008%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34276/134739 objects degraded (25.439%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:04.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:04.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:04.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.779 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:04.779 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:04.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:04.796 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:04.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:04.811 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:04.812 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:04.827 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:04.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:04.843 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:04.843 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:04.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:04.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:04.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:04.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:05.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:05.543 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:05.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:05.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:05 smithi017 ceph-mon[107556]: from='client.36561 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:05.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4261928078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:05.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:05 smithi017 ceph-mon[103001]: from='client.36561 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:05.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4261928078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:05.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:05.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:05 smithi189 ceph-mon[90992]: from='client.36561 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4261928078' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:05.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:05.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:05.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:06.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:06.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:06.128 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1147,"stamp":"2024-09-13T14:47:03.251321+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2946206390,"num_objects":44913,"num_object_clones":0,"num_object_copies":134739,"num_objects_missing_on_primary":1754,"num_objects_missing":1754,"num_objects_degraded":34276,"num_objects_misplaced":2705,"num_objects_unfound":0,"num_objects_dirty":44913,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":138740,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25673,"num_bytes_recovered":1618512029,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126025,"ondisk_log_size":126025,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13315280,"kb_used_data":8264656,"kb_used_omap":44,"kb_used_meta":5050515,"kb_avail":736645936,"statfs":{"total":767960285184,"available":754325438464,"internally_reserved":0,"allocated":8463007744,"data_stored":8447789159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5171727607},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-120193024,"num_objects":-1834,"num_object_clones":0,"num_object_copies":-5502,"num_objects_missing_on_primary":-110,"num_objects_missing":-110,"num_objects_degraded":-414,"num_objects_misplaced":-114,"num_objects_unfound":0,"num_objects_dirty":-1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1834,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.555182"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":604,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.728123+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:46:51.728123+0000","last_peered":"2024-09-13T14:46:51.728123+0000","last_clean":"2024-09-13T14:46:51.728123+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:46:51.728123+0000","last_undegraded":"2024-09-13T14:46:51.728123+0000","last_fullsized":"2024-09-13T14:46:51.728123+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":358,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372933+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:46:36.372933+0000","last_peered":"2024-09-13T14:46:36.372933+0000","last_clean":"2024-09-13T14:46:36.372933+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:46:36.372933+0000","last_undegraded":"2024-09-13T14:46:36.372933+0000","last_fullsized":"2024-09-13T14:46:36.372933+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":287,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.369327+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:46:36.369327+0000","last_peered":"2024-09-13T14:46:36.369327+0000","last_clean":"2024-09-13T14:46:36.369327+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:46:36.369327+0000","last_undegraded":"2024-09-13T14:46:36.369327+0000","last_fullsized":"2024-09-13T14:46:36.369327+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":589,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704287+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:46:51.704287+0000","last_peered":"2024-09-13T14:46:51.704287+0000","last_clean":"2024-09-13T14:46:51.704287+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:46:51.704287+0000","last_undegraded":"2024-09-13T14:46:51.704287+0000","last_fullsized":"2024-09-13T14:46:51.704287+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":590,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.705389+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:46:51.705389+0000","last_peered":"2024-09-13T14:46:51.705389+0000","last_clean":"2024-09-13T14:46:51.705389+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:46:51.705389+0000","last_undegraded":"2024-09-13T14:46:51.705389+0000","last_fullsized":"2024-09-13T14:46:51.705389+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":327,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372757+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:46:36.372757+0000","last_peered":"2024-09-13T14:46:36.372757+0000","last_clean":"2024-09-13T14:46:36.372757+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:46:36.372757+0000","last_undegraded":"2024-09-13T14:46:36.372757+0000","last_fullsized":"2024-09-13T14:46:36.372757+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":604,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723300+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:46:51.723300+0000","last_peered":"2024-09-13T14:46:51.723300+0000","last_clean":"2024-09-13T14:46:51.723300+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:46:51.723300+0000","last_undegraded":"2024-09-13T14:46:51.723300+0000","last_fullsized":"2024-09-13T14:46:51.723300+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":349,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366462+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:46:36.366462+0000","last_peered":"2024-09-13T14:46:36.366462+0000","last_clean":"2024-09-13T14:46:36.366462+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:46:36.366462+0000","last_undegraded":"2024-09-13T14:46:36.366462+0000","last_fullsized":"2024-09-13T14:46:36.366462+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"360'8566","reported_seq":17451,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:01.807625+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:47:01.807625+0000","last_peered":"2024-09-13T14:47:01.807625+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:47:01.807625+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8566,"log_dups_size":0,"ondisk_log_size":8566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8566,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2921,"num_bytes_recovered":191430656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1740}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":353,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374434+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:46:36.374434+0000","last_peered":"2024-09-13T14:46:36.374434+0000","last_clean":"2024-09-13T14:46:36.374434+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:46:36.374434+0000","last_undegraded":"2024-09-13T14:46:36.374434+0000","last_fullsized":"2024-09-13T14:46:36.374434+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":272,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367489+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:46:36.367489+0000","last_peered":"2024-09-13T14:46:36.367489+0000","last_clean":"2024-09-13T14:46:36.367489+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:46:36.367489+0000","last_undegraded":"2024-09-13T14:46:36.367489+0000","last_fullsized":"2024-09-13T14:46:36.367489+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"360'8684","reported_seq":18241,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:47:03.094168+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:47:03.094168+0000","last_peered":"2024-09-13T14:47:03.094168+0000","last_clean":"2024-09-13T14:47:03.094168+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:47:03.094168+0000","last_undegraded":"2024-09-13T14:47:03.094168+0000","last_fullsized":"2024-09-13T14:47:03.094168+0000","mapping_epoch":326,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183762944,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8684,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":371,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374580+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:46:36.374580+0000","last_peered":"2024-09-13T14:46:36.374580+0000","last_clean":"2024-09-13T14:46:36.374580+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:46:36.374580+0000","last_undegraded":"2024-09-13T14:46:36.374580+0000","last_fullsized":"2024-09-13T14:46:36.374580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373265+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:46:36.373265+0000","last_peered":"2024-09-13T14:46:36.373265+0000","last_clean":"2024-09-13T14:46:36.373265+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:46:36.373265+0000","last_undegraded":"2024-09-13T14:46:36.373265+0000","last_fullsized":"2024-09-13T14:46:36.373265+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"360'8934","reported_seq":9062,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:58.532085+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:46:58.532085+0000","last_peered":"2024-09-13T14:46:58.532085+0000","last_clean":"2024-09-13T14:46:58.532085+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:46:58.532085+0000","last_undegraded":"2024-09-13T14:46:58.532085+0000","last_fullsized":"2024-09-13T14:46:58.532085+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8634,"log_dups_size":0,"ondisk_log_size":8634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187695104,"num_objects":2864,"num_object_clones":0,"num_object_copies":8592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8934,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":325,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374904+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:46:36.374904+0000","last_peered":"2024-09-13T14:46:36.374904+0000","last_clean":"2024-09-13T14:46:36.374904+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:46:36.374904+0000","last_undegraded":"2024-09-13T14:46:36.374904+0000","last_fullsized":"2024-09-13T14:46:36.374904+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":340,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367108+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:46:36.367108+0000","last_peered":"2024-09-13T14:46:36.367108+0000","last_clean":"2024-09-13T14:46:36.367108+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:46:36.367108+0000","last_undegraded":"2024-09-13T14:46:36.367108+0000","last_fullsized":"2024-09-13T14:46:36.367108+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"360'8706","reported_seq":8834,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:59.939997+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:46:59.939997+0000","last_peered":"2024-09-13T14:46:59.939997+0000","last_clean":"2024-09-13T14:46:59.939997+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:46:59.939997+0000","last_undegraded":"2024-09-13T14:46:59.939997+0000","last_fullsized":"2024-09-13T14:46:59.939997+0000","mapping_epoch":303,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7506,"log_dups_size":0,"ondisk_log_size":7506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":288,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367135+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:46:36.367135+0000","last_peered":"2024-09-13T14:46:36.367135+0000","last_clean":"2024-09-13T14:46:36.367135+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:46:36.367135+0000","last_undegraded":"2024-09-13T14:46:36.367135+0000","last_fullsized":"2024-09-13T14:46:36.367135+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":616,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704432+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:46:51.704432+0000","last_peered":"2024-09-13T14:46:51.704432+0000","last_clean":"2024-09-13T14:46:51.704432+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:46:51.704432+0000","last_undegraded":"2024-09-13T14:46:51.704432+0000","last_fullsized":"2024-09-13T14:46:51.704432+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"360'8679","reported_seq":8804,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:00.434222+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:47:00.434222+0000","last_peered":"2024-09-13T14:47:00.434222+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:47:00.434222+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182386688,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8679,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2850}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":471,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372574+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:46:36.372574+0000","last_peered":"2024-09-13T14:46:36.372574+0000","last_clean":"2024-09-13T14:46:36.372574+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:46:36.372574+0000","last_undegraded":"2024-09-13T14:46:36.372574+0000","last_fullsized":"2024-09-13T14:46:36.372574+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":614,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723172+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:46:51.723172+0000","last_peered":"2024-09-13T14:46:51.723172+0000","last_clean":"2024-09-13T14:46:51.723172+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:46:51.723172+0000","last_undegraded":"2024-09-13T14:46:51.723172+0000","last_fullsized":"2024-09-13T14:46:51.723172+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373893+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:46:36.373893+0000","last_peered":"2024-09-13T14:46:36.373893+0000","last_clean":"2024-09-13T14:46:36.373893+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:46:36.373893+0000","last_undegraded":"2024-09-13T14:46:36.373893+0000","last_fullsized":"2024-09-13T14:46:36.373893+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"360'8335","reported_seq":8463,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:59.960034+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:46:59.960034+0000","last_peered":"2024-09-13T14:46:59.960034+0000","last_clean":"2024-09-13T14:46:59.960034+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:46:59.960034+0000","last_undegraded":"2024-09-13T14:46:59.960034+0000","last_fullsized":"2024-09-13T14:46:59.960034+0000","mapping_epoch":303,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8335,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":175,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367559+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:46:36.367559+0000","last_peered":"2024-09-13T14:46:36.367559+0000","last_clean":"2024-09-13T14:46:36.367559+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:46:36.367559+0000","last_undegraded":"2024-09-13T14:46:36.367559+0000","last_fullsized":"2024-09-13T14:46:36.367559+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":361,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366977+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:46:36.366977+0000","last_peered":"2024-09-13T14:46:36.366977+0000","last_clean":"2024-09-13T14:46:36.366977+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:46:36.366977+0000","last_undegraded":"2024-09-13T14:46:36.366977+0000","last_fullsized":"2024-09-13T14:46:36.366977+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":73,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373666+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:46:36.373666+0000","last_peered":"2024-09-13T14:46:36.373666+0000","last_clean":"2024-09-13T14:46:36.373666+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:46:36.373666+0000","last_undegraded":"2024-09-13T14:46:36.373666+0000","last_fullsized":"2024-09-13T14:46:36.373666+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"360'8604","reported_seq":17017,"reported_epoch":360,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:00.032608+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:47:00.032608+0000","last_peered":"2024-09-13T14:47:00.032608+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:47:00.032608+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":1754,"num_objects_missing":1754,"num_objects_degraded":2129,"num_objects_misplaced":2705,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8604,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2898,"num_bytes_recovered":189923328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1754},{"shards":"0,3,4","objects":951},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373093+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:46:36.373093+0000","last_peered":"2024-09-13T14:46:36.373093+0000","last_clean":"2024-09-13T14:46:36.373093+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:46:36.373093+0000","last_undegraded":"2024-09-13T14:46:36.373093+0000","last_fullsized":"2024-09-13T14:46:36.373093+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373800+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:46:36.373800+0000","last_peered":"2024-09-13T14:46:36.373800+0000","last_clean":"2024-09-13T14:46:36.373800+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:46:36.373800+0000","last_undegraded":"2024-09-13T14:46:36.373800+0000","last_fullsized":"2024-09-13T14:46:36.373800+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"360'8388","reported_seq":8516,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:59.865233+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:46:59.865233+0000","last_peered":"2024-09-13T14:46:59.865233+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:46:59.865233+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7588,"log_dups_size":0,"ondisk_log_size":7588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181534720,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8388,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2696}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":168,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366856+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:46:36.366856+0000","last_peered":"2024-09-13T14:46:36.366856+0000","last_clean":"2024-09-13T14:46:36.366856+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:46:36.366856+0000","last_undegraded":"2024-09-13T14:46:36.366856+0000","last_fullsized":"2024-09-13T14:46:36.366856+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":606,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.730424+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:46:51.730424+0000","last_peered":"2024-09-13T14:46:51.730424+0000","last_clean":"2024-09-13T14:46:51.730424+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:46:51.730424+0000","last_undegraded":"2024-09-13T14:46:51.730424+0000","last_fullsized":"2024-09-13T14:46:51.730424+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":568,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698109+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:46:51.698109+0000","last_peered":"2024-09-13T14:46:51.698109+0000","last_clean":"2024-09-13T14:46:51.698109+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:46:51.698109+0000","last_undegraded":"2024-09-13T14:46:51.698109+0000","last_fullsized":"2024-09-13T14:46:51.698109+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"360'8461","reported_seq":8533,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:02.828569+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:47:02.828569+0000","last_peered":"2024-09-13T14:47:02.828569+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:47:02.828569+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8461,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2737}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":462,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372987+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:46:36.372987+0000","last_peered":"2024-09-13T14:46:36.372987+0000","last_clean":"2024-09-13T14:46:36.372987+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:46:36.372987+0000","last_undegraded":"2024-09-13T14:46:36.372987+0000","last_fullsized":"2024-09-13T14:46:36.372987+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":884,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367847+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:46:36.367847+0000","last_peered":"2024-09-13T14:46:36.367847+0000","last_clean":"2024-09-13T14:46:36.367847+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:46:36.367847+0000","last_undegraded":"2024-09-13T14:46:36.367847+0000","last_fullsized":"2024-09-13T14:46:36.367847+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":560,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723263+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:46:51.723263+0000","last_peered":"2024-09-13T14:46:51.723263+0000","last_clean":"2024-09-13T14:46:51.723263+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:46:51.723263+0000","last_undegraded":"2024-09-13T14:46:51.723263+0000","last_fullsized":"2024-09-13T14:46:51.723263+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":326,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.371179+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:46:36.371179+0000","last_peered":"2024-09-13T14:46:36.371179+0000","last_clean":"2024-09-13T14:46:36.371179+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:46:36.371179+0000","last_undegraded":"2024-09-13T14:46:36.371179+0000","last_fullsized":"2024-09-13T14:46:36.371179+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"360'8728","reported_seq":8854,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:46:59.947263+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:46:59.947263+0000","last_peered":"2024-09-13T14:46:59.947263+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:46:59.947263+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184811520,"num_objects":2820,"num_object_clones":0,"num_object_copies":8460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8728,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-2860}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":413,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.701844+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:46:51.701844+0000","last_peered":"2024-09-13T14:46:51.701844+0000","last_clean":"2024-09-13T14:46:51.701844+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:46:51.701844+0000","last_undegraded":"2024-09-13T14:46:51.701844+0000","last_fullsized":"2024-09-13T14:46:51.701844+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":750,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372962+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:46:36.372962+0000","last_peered":"2024-09-13T14:46:36.372962+0000","last_clean":"2024-09-13T14:46:36.372962+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:46:36.372962+0000","last_undegraded":"2024-09-13T14:46:36.372962+0000","last_fullsized":"2024-09-13T14:46:36.372962+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":609,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.703165+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:46:51.703165+0000","last_peered":"2024-09-13T14:46:51.703165+0000","last_clean":"2024-09-13T14:46:51.703165+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:46:51.703165+0000","last_undegraded":"2024-09-13T14:46:51.703165+0000","last_fullsized":"2024-09-13T14:46:51.703165+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":613,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.705504+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:46:51.705504+0000","last_peered":"2024-09-13T14:46:51.705504+0000","last_clean":"2024-09-13T14:46:51.705504+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:46:51.705504+0000","last_undegraded":"2024-09-13T14:46:51.705504+0000","last_fullsized":"2024-09-13T14:46:51.705504+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"360'8731","reported_seq":18400,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:47:00.441711+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:47:00.441711+0000","last_peered":"2024-09-13T14:47:00.441711+0000","last_clean":"2024-09-13T14:47:00.441711+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:47:00.441711+0000","last_undegraded":"2024-09-13T14:47:00.441711+0000","last_fullsized":"2024-09-13T14:47:00.441711+0000","mapping_epoch":330,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192217088,"num_objects":2933,"num_object_clones":0,"num_object_copies":8799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8731,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":491,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372644+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:46:36.372644+0000","last_peered":"2024-09-13T14:46:36.372644+0000","last_clean":"2024-09-13T14:46:36.372644+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:46:36.372644+0000","last_undegraded":"2024-09-13T14:46:36.372644+0000","last_fullsized":"2024-09-13T14:46:36.372644+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":577,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.722811+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:46:51.722811+0000","last_peered":"2024-09-13T14:46:51.722811+0000","last_clean":"2024-09-13T14:46:51.722811+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:46:51.722811+0000","last_undegraded":"2024-09-13T14:46:51.722811+0000","last_fullsized":"2024-09-13T14:46:51.722811+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":546,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697583+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:46:51.697583+0000","last_peered":"2024-09-13T14:46:51.697583+0000","last_clean":"2024-09-13T14:46:51.697583+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:46:51.697583+0000","last_undegraded":"2024-09-13T14:46:51.697583+0000","last_fullsized":"2024-09-13T14:46:51.697583+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.369287+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:46:36.369287+0000","last_peered":"2024-09-13T14:46:36.369287+0000","last_clean":"2024-09-13T14:46:36.369287+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:46:36.369287+0000","last_undegraded":"2024-09-13T14:46:36.369287+0000","last_fullsized":"2024-09-13T14:46:36.369287+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"360'8411","reported_seq":8538,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:00.506391+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:47:00.506391+0000","last_peered":"2024-09-13T14:47:00.506391+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:47:00.506391+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184745984,"num_objects":2819,"num_object_clones":0,"num_object_copies":8457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8411,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2690}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":168,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366556+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:46:36.366556+0000","last_peered":"2024-09-13T14:46:36.366556+0000","last_clean":"2024-09-13T14:46:36.366556+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:46:36.366556+0000","last_undegraded":"2024-09-13T14:46:36.366556+0000","last_fullsized":"2024-09-13T14:46:36.366556+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":975,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366924+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:46:36.366924+0000","last_peered":"2024-09-13T14:46:36.366924+0000","last_clean":"2024-09-13T14:46:36.366924+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:46:36.366924+0000","last_undegraded":"2024-09-13T14:46:36.366924+0000","last_fullsized":"2024-09-13T14:46:36.366924+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373792+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:46:36.373792+0000","last_peered":"2024-09-13T14:46:36.373792+0000","last_clean":"2024-09-13T14:46:36.373792+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:46:36.373792+0000","last_undegraded":"2024-09-13T14:46:36.373792+0000","last_fullsized":"2024-09-13T14:46:36.373792+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":566,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697736+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:46:51.697736+0000","last_peered":"2024-09-13T14:46:51.697736+0000","last_clean":"2024-09-13T14:46:51.697736+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:46:51.697736+0000","last_undegraded":"2024-09-13T14:46:51.697736+0000","last_fullsized":"2024-09-13T14:46:51.697736+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"360'8564","reported_seq":17972,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:58.536681+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:46:58.536681+0000","last_peered":"2024-09-13T14:46:58.536681+0000","last_clean":"2024-09-13T14:46:58.536681+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:46:58.536681+0000","last_undegraded":"2024-09-13T14:46:58.536681+0000","last_fullsized":"2024-09-13T14:46:58.536681+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8564,"log_dups_size":0,"ondisk_log_size":8564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8564,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":359,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366853+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:46:36.366853+0000","last_peered":"2024-09-13T14:46:36.366853+0000","last_clean":"2024-09-13T14:46:36.366853+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:46:36.366853+0000","last_undegraded":"2024-09-13T14:46:36.366853+0000","last_fullsized":"2024-09-13T14:46:36.366853+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":305,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.369152+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:46:36.369152+0000","last_peered":"2024-09-13T14:46:36.369152+0000","last_clean":"2024-09-13T14:46:36.369152+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:46:36.369152+0000","last_undegraded":"2024-09-13T14:46:36.369152+0000","last_fullsized":"2024-09-13T14:46:36.369152+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1145,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697445+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:46:51.697445+0000","last_peered":"2024-09-13T14:46:51.697445+0000","last_clean":"2024-09-13T14:46:51.697445+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:46:51.697445+0000","last_undegraded":"2024-09-13T14:46:51.697445+0000","last_fullsized":"2024-09-13T14:46:51.697445+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":547,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700228+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:46:51.700228+0000","last_peered":"2024-09-13T14:46:51.700228+0000","last_clean":"2024-09-13T14:46:51.700228+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:46:51.700228+0000","last_undegraded":"2024-09-13T14:46:51.700228+0000","last_fullsized":"2024-09-13T14:46:51.700228+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"360'8447","reported_seq":8575,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:47:00.411862+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:47:00.411862+0000","last_peered":"2024-09-13T14:47:00.411862+0000","last_clean":"2024-09-13T14:47:00.411862+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:47:00.411862+0000","last_undegraded":"2024-09-13T14:47:00.411862+0000","last_fullsized":"2024-09-13T14:47:00.411862+0000","mapping_epoch":303,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7047,"log_dups_size":0,"ondisk_log_size":7047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180682752,"num_objects":2757,"num_object_clones":0,"num_object_copies":8271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8447,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":359,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367307+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:46:36.367307+0000","last_peered":"2024-09-13T14:46:36.367307+0000","last_clean":"2024-09-13T14:46:36.367307+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:46:36.367307+0000","last_undegraded":"2024-09-13T14:46:36.367307+0000","last_fullsized":"2024-09-13T14:46:36.367307+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":556,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697326+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:46:51.697326+0000","last_peered":"2024-09-13T14:46:51.697326+0000","last_clean":"2024-09-13T14:46:51.697326+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:46:51.697326+0000","last_undegraded":"2024-09-13T14:46:51.697326+0000","last_fullsized":"2024-09-13T14:46:51.697326+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":754,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374876+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:46:36.374876+0000","last_peered":"2024-09-13T14:46:36.374876+0000","last_clean":"2024-09-13T14:46:36.374876+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:46:36.374876+0000","last_undegraded":"2024-09-13T14:46:36.374876+0000","last_fullsized":"2024-09-13T14:46:36.374876+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":592,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704357+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:46:51.704357+0000","last_peered":"2024-09-13T14:46:51.704357+0000","last_clean":"2024-09-13T14:46:51.704357+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:46:51.704357+0000","last_undegraded":"2024-09-13T14:46:51.704357+0000","last_fullsized":"2024-09-13T14:46:51.704357+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"360'8814","reported_seq":8942,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:58.734846+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:46:58.734846+0000","last_peered":"2024-09-13T14:46:58.734846+0000","last_clean":"2024-09-13T14:46:58.734846+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:46:58.734846+0000","last_undegraded":"2024-09-13T14:46:58.734846+0000","last_fullsized":"2024-09-13T14:46:58.734846+0000","mapping_epoch":303,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8614,"log_dups_size":0,"ondisk_log_size":8614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188481536,"num_objects":2876,"num_object_clones":0,"num_object_copies":8628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8814,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373485+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:46:36.373485+0000","last_peered":"2024-09-13T14:46:36.373485+0000","last_clean":"2024-09-13T14:46:36.373485+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:46:36.373485+0000","last_undegraded":"2024-09-13T14:46:36.373485+0000","last_fullsized":"2024-09-13T14:46:36.373485+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":592,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723536+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:46:51.723536+0000","last_peered":"2024-09-13T14:46:51.723536+0000","last_clean":"2024-09-13T14:46:51.723536+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:46:51.723536+0000","last_undegraded":"2024-09-13T14:46:51.723536+0000","last_fullsized":"2024-09-13T14:46:51.723536+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":932,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697756+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:46:51.697756+0000","last_peered":"2024-09-13T14:46:51.697756+0000","last_clean":"2024-09-13T14:46:51.697756+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:46:51.697756+0000","last_undegraded":"2024-09-13T14:46:51.697756+0000","last_fullsized":"2024-09-13T14:46:51.697756+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":590,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.730329+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:46:51.730329+0000","last_peered":"2024-09-13T14:46:51.730329+0000","last_clean":"2024-09-13T14:46:51.730329+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:46:51.730329+0000","last_undegraded":"2024-09-13T14:46:51.730329+0000","last_fullsized":"2024-09-13T14:46:51.730329+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"360'8741","reported_seq":20112,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:47:02.650442+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:47:02.650442+0000","last_peered":"2024-09-13T14:47:02.650442+0000","last_clean":"2024-09-13T14:47:02.650442+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:47:02.650442+0000","last_undegraded":"2024-09-13T14:47:02.650442+0000","last_fullsized":"2024-09-13T14:47:02.650442+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8741,"log_dups_size":0,"ondisk_log_size":8741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181665816,"num_objects":2773,"num_object_clones":0,"num_object_copies":8319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8741,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":444,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.728022+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:46:51.728022+0000","last_peered":"2024-09-13T14:46:51.728022+0000","last_clean":"2024-09-13T14:46:51.728022+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:46:51.728022+0000","last_undegraded":"2024-09-13T14:46:51.728022+0000","last_fullsized":"2024-09-13T14:46:51.728022+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":627,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.705389+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:46:51.705389+0000","last_peered":"2024-09-13T14:46:51.705389+0000","last_clean":"2024-09-13T14:46:51.705389+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:46:51.705389+0000","last_undegraded":"2024-09-13T14:46:51.705389+0000","last_fullsized":"2024-09-13T14:46:51.705389+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":778,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366982+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:46:36.366982+0000","last_peered":"2024-09-13T14:46:36.366982+0000","last_clean":"2024-09-13T14:46:36.366982+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:46:36.366982+0000","last_undegraded":"2024-09-13T14:46:36.366982+0000","last_fullsized":"2024-09-13T14:46:36.366982+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373175+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:46:36.373175+0000","last_peered":"2024-09-13T14:46:36.373175+0000","last_clean":"2024-09-13T14:46:36.373175+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:46:36.373175+0000","last_undegraded":"2024-09-13T14:46:36.373175+0000","last_fullsized":"2024-09-13T14:46:36.373175+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373937+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:46:36.373937+0000","last_peered":"2024-09-13T14:46:36.373937+0000","last_clean":"2024-09-13T14:46:36.373937+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:46:36.373937+0000","last_undegraded":"2024-09-13T14:46:36.373937+0000","last_fullsized":"2024-09-13T14:46:36.373937+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":69,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373285+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:46:36.373285+0000","last_peered":"2024-09-13T14:46:36.373285+0000","last_clean":"2024-09-13T14:46:36.373285+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:46:36.373285+0000","last_undegraded":"2024-09-13T14:46:36.373285+0000","last_fullsized":"2024-09-13T14:46:36.373285+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":544,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700077+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:46:51.700077+0000","last_peered":"2024-09-13T14:46:51.700077+0000","last_clean":"2024-09-13T14:46:51.700077+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:46:51.700077+0000","last_undegraded":"2024-09-13T14:46:51.700077+0000","last_fullsized":"2024-09-13T14:46:51.700077+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":359,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.374775+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:46:36.374775+0000","last_peered":"2024-09-13T14:46:36.374775+0000","last_clean":"2024-09-13T14:46:36.374775+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:46:36.374775+0000","last_undegraded":"2024-09-13T14:46:36.374775+0000","last_fullsized":"2024-09-13T14:46:36.374775+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":325,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.369286+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:46:36.369286+0000","last_peered":"2024-09-13T14:46:36.369286+0000","last_clean":"2024-09-13T14:46:36.369286+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:46:36.369286+0000","last_undegraded":"2024-09-13T14:46:36.369286+0000","last_fullsized":"2024-09-13T14:46:36.369286+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":548,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.722583+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:46:51.722583+0000","last_peered":"2024-09-13T14:46:51.722583+0000","last_clean":"2024-09-13T14:46:51.722583+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:46:51.722583+0000","last_undegraded":"2024-09-13T14:46:51.722583+0000","last_fullsized":"2024-09-13T14:46:51.722583+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":600,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704606+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:46:51.704606+0000","last_peered":"2024-09-13T14:46:51.704606+0000","last_clean":"2024-09-13T14:46:51.704606+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:46:51.704606+0000","last_undegraded":"2024-09-13T14:46:51.704606+0000","last_fullsized":"2024-09-13T14:46:51.704606+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":604,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.730312+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:46:51.730312+0000","last_peered":"2024-09-13T14:46:51.730312+0000","last_clean":"2024-09-13T14:46:51.730312+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:46:51.730312+0000","last_undegraded":"2024-09-13T14:46:51.730312+0000","last_fullsized":"2024-09-13T14:46:51.730312+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":357,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.366918+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:46:36.366918+0000","last_peered":"2024-09-13T14:46:36.366918+0000","last_clean":"2024-09-13T14:46:36.366918+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:46:36.366918+0000","last_undegraded":"2024-09-13T14:46:36.366918+0000","last_fullsized":"2024-09-13T14:46:36.366918+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":622,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.730413+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:46:51.730413+0000","last_peered":"2024-09-13T14:46:51.730413+0000","last_clean":"2024-09-13T14:46:51.730413+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:46:51.730413+0000","last_undegraded":"2024-09-13T14:46:51.730413+0000","last_fullsized":"2024-09-13T14:46:51.730413+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":592,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.705499+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:46:51.705499+0000","last_peered":"2024-09-13T14:46:51.705499+0000","last_clean":"2024-09-13T14:46:51.705499+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:46:51.705499+0000","last_undegraded":"2024-09-13T14:46:51.705499+0000","last_fullsized":"2024-09-13T14:46:51.705499+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":569,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700465+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:46:51.700465+0000","last_peered":"2024-09-13T14:46:51.700465+0000","last_clean":"2024-09-13T14:46:51.700465+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:46:51.700465+0000","last_undegraded":"2024-09-13T14:46:51.700465+0000","last_fullsized":"2024-09-13T14:46:51.700465+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":618,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.722634+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:46:51.722634+0000","last_peered":"2024-09-13T14:46:51.722634+0000","last_clean":"2024-09-13T14:46:51.722634+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:46:51.722634+0000","last_undegraded":"2024-09-13T14:46:51.722634+0000","last_fullsized":"2024-09-13T14:46:51.722634+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":58,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.399538+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:46:36.399538+0000","last_peered":"2024-09-13T14:46:36.399538+0000","last_clean":"2024-09-13T14:46:36.399538+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:46:36.399538+0000","last_undegraded":"2024-09-13T14:46:36.399538+0000","last_fullsized":"2024-09-13T14:46:36.399538+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":515,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.697624+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:46:51.697624+0000","last_peered":"2024-09-13T14:46:51.697624+0000","last_clean":"2024-09-13T14:46:51.697624+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:46:51.697624+0000","last_undegraded":"2024-09-13T14:46:51.697624+0000","last_fullsized":"2024-09-13T14:46:51.697624+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":596,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704261+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:46:51.704261+0000","last_peered":"2024-09-13T14:46:51.704261+0000","last_clean":"2024-09-13T14:46:51.704261+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:46:51.704261+0000","last_undegraded":"2024-09-13T14:46:51.704261+0000","last_fullsized":"2024-09-13T14:46:51.704261+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.373728+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:46:36.373728+0000","last_peered":"2024-09-13T14:46:36.373728+0000","last_clean":"2024-09-13T14:46:36.373728+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:46:36.373728+0000","last_undegraded":"2024-09-13T14:46:36.373728+0000","last_fullsized":"2024-09-13T14:46:36.373728+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":325,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.367200+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:46:36.367200+0000","last_peered":"2024-09-13T14:46:36.367200+0000","last_clean":"2024-09-13T14:46:36.367200+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:46:36.367200+0000","last_undegraded":"2024-09-13T14:46:36.367200+0000","last_fullsized":"2024-09-13T14:46:36.367200+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":58,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.399490+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:46:36.399490+0000","last_peered":"2024-09-13T14:46:36.399490+0000","last_clean":"2024-09-13T14:46:36.399490+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:46:36.399490+0000","last_undegraded":"2024-09-13T14:46:36.399490+0000","last_fullsized":"2024-09-13T14:46:36.399490+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":291,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.369388+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:46:36.369388+0000","last_peered":"2024-09-13T14:46:36.369388+0000","last_clean":"2024-09-13T14:46:36.369388+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:46:36.369388+0000","last_undegraded":"2024-09-13T14:46:36.369388+0000","last_fullsized":"2024-09-13T14:46:36.369388+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":614,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723402+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:46:51.723402+0000","last_peered":"2024-09-13T14:46:51.723402+0000","last_clean":"2024-09-13T14:46:51.723402+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:46:51.723402+0000","last_undegraded":"2024-09-13T14:46:51.723402+0000","last_fullsized":"2024-09-13T14:46:51.723402+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":601,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.723077+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:46:51.723077+0000","last_peered":"2024-09-13T14:46:51.723077+0000","last_clean":"2024-09-13T14:46:51.723077+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:46:51.723077+0000","last_undegraded":"2024-09-13T14:46:51.723077+0000","last_fullsized":"2024-09-13T14:46:51.723077+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-09-13T14:46:36.372554+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:46:36.372554+0000","last_peered":"2024-09-13T14:46:36.372554+0000","last_clean":"2024-09-13T14:46:36.372554+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:46:36.372554+0000","last_undegraded":"2024-09-13T14:46:36.372554+0000","last_fullsized":"2024-09-13T14:46:36.372554+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":606,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.704521+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:46:51.704521+0000","last_peered":"2024-09-13T14:46:51.704521+0000","last_clean":"2024-09-13T14:46:51.704521+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:46:51.704521+0000","last_undegraded":"2024-09-13T14:46:51.704521+0000","last_fullsized":"2024-09-13T14:46:51.704521+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2941517848,"num_objects":44885,"num_object_clones":0,"num_object_copies":134655,"num_objects_missing_on_primary":1754,"num_objects_missing":1754,"num_objects_degraded":34276,"num_objects_misplaced":2705,"num_objects_unfound":0,"num_objects_dirty":44885,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137793,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25616,"num_bytes_recovered":1618345984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8395501568,"data_stored":8395489352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123893,"ondisk_log_size":123893,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397931,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810364,"kb_used_data":910196,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91934788,"statfs":{"total":95995035648,"available":94141222912,"internally_reserved":0,"allocated":932040704,"data_stored":930122825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594159,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848808,"kb_used_data":1254752,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":91896344,"statfs":{"total":95995035648,"available":94101856256,"internally_reserved":0,"allocated":1284866048,"data_stored":1282967923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665613,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":566120,"kb_used_data":229408,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93179032,"statfs":{"total":95995035648,"available":95415328768,"internally_reserved":0,"allocated":234913792,"data_stored":233011998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986614,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1591668,"kb_used_data":683500,"kb_used_omap":1,"kb_used_meta":908158,"kb_avail":92153484,"statfs":{"total":95995035648,"available":94365167616,"internally_reserved":0,"allocated":699904000,"data_stored":697994219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":929954136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697174,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208164,"kb_used_data":1454172,"kb_used_omap":4,"kb_used_meta":753979,"kb_avail":91536988,"statfs":{"total":95995035648,"available":93733875712,"internally_reserved":0,"allocated":1489072128,"data_stored":1487174487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":772074571},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860701,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989804,"kb_used_data":1368548,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":91755348,"statfs":{"total":95995035648,"available":93957476352,"internally_reserved":0,"allocated":1401393152,"data_stored":1399487857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020372,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584608,"kb_used_data":1271320,"kb_used_omap":9,"kb_used_meta":313270,"kb_avail":92160544,"statfs":{"total":95995035648,"available":94372397056,"internally_reserved":0,"allocated":1301831680,"data_stored":1299937512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":320789317},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187762,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1715744,"kb_used_data":1092760,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":92029408,"statfs":{"total":95995035648,"available":94238113792,"internally_reserved":0,"allocated":1118986240,"data_stored":1117092338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1109262336,"data_stored":1109262336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1289289728,"data_stored":1289289728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1396445184,"data_stored":1396441112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1480523776,"data_stored":1480523776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":694288384,"data_stored":694288384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":229511168,"data_stored":229507096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1276837888,"data_stored":1276837888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":919343104,"data_stored":919339032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:47:06.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:47:06.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:06.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:06.411 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:06.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:06 smithi017 ceph-mon[103001]: pgmap v1148: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34218/133518 objects degraded (25.628%); 2649/133518 objects misplaced (1.984%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:06 smithi017 ceph-mon[107556]: pgmap v1148: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34218/133518 objects degraded (25.628%); 2649/133518 objects misplaced (1.984%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:06.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:06.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:06.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:06 smithi189 ceph-mon[90992]: pgmap v1148: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 34218/133518 objects degraded (25.628%); 2649/133518 objects misplaced (1.984%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:06.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:06.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:06.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:07.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:07.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:07.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:07.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:07.323 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:47:07.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3774668877' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:07 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:07] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-13T14:47:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3774668877' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:07 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:07.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:07.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:07.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:07.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:07.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3774668877' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:07 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:07.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:07.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:07.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:08.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:08.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:08.209 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:08.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:08.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[103001]: pgmap v1149: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 34138/132783 objects degraded (25.710%); 2649/132783 objects misplaced (1.995%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[103001]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:47:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[107556]: pgmap v1149: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 34138/132783 objects degraded (25.710%); 2649/132783 objects misplaced (1.995%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:08 smithi017 ceph-mon[107556]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:47:08.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:08.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:08.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:08.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:08.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:08.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:08.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:08 smithi189 ceph-mon[90992]: pgmap v1149: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 34138/132783 objects degraded (25.710%); 2649/132783 objects misplaced (1.995%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:08 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:08 smithi189 ceph-mon[90992]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:47:08.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:08.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:09.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:09.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:09.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:09.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:09.561 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 34138/132783 objects degraded (25.710%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:09.561 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 34138/132783 objects degraded (25.710%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:09.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:09.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:09.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.721 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:09.721 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 34138/131685 objects degraded (25.924%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1627,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":361,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":43895,"data_bytes":2879490742,"bytes_used":13473603584,"bytes_avail":754486681600,"bytes_total":767960285184,"degraded_objects":34138,"degraded_total":131685,"degraded_ratio":0.25923985267874095,"misplaced_objects":2649,"misplaced_total":131685,"misplaced_ratio":0.020116186353798839,"recovering_objects_per_sec":20,"recovering_bytes_per_sec":1316921,"recovering_keys_per_sec":0,"num_objects_recovered":201,"num_bytes_recovered":13172736,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":161},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (8m)\n [==========================..] (remaining: 36s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:47:09.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 34138/132783 objects degraded (25.710%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:09.836 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:09.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:09.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:10.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:10.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:47:10.128 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:10.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.414 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:10 smithi017 ceph-mon[103001]: pgmap v1151: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 34138/131685 objects degraded (25.924%); 2649/131685 objects misplaced (2.012%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:10.414 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1958812444' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:10.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:10 smithi017 ceph-mon[107556]: pgmap v1151: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 34138/131685 objects degraded (25.924%); 2649/131685 objects misplaced (2.012%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:10.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1958812444' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:10.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:10.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:10 smithi189 ceph-mon[90992]: pgmap v1151: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 34138/131685 objects degraded (25.924%); 2649/131685 objects misplaced (2.012%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:10.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1958812444' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:10.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:10.865 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:10.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:10.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:11.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:11.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:11.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:11.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:11.255 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:47:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=cleanup t=2024-09-13T14:47:10.943687159Z level=info msg="Completed cleanup jobs" duration=140.746739ms 2024-09-13T14:47:11.342 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:11.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:11.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:11.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:11.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:11.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:11.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:11.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:12.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:12.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:12.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:12.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:12.235 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:12.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:12.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.452 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:47:12.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:47:12.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:12.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:12.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:12.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:12.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:12 smithi017 ceph-mon[103001]: pgmap v1152: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 34080/130095 objects degraded (26.196%); 2591/130095 objects misplaced (1.992%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:12.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:12.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:12 smithi017 ceph-mon[107556]: pgmap v1152: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 34080/130095 objects degraded (26.196%); 2591/130095 objects misplaced (1.992%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:12.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:12.660 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:47:12.660 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:47:12.660 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:47:12.660 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:47:12.661 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:47:12.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:12 smithi189 ceph-mon[90992]: pgmap v1152: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 34080/130095 objects degraded (26.196%); 2591/130095 objects misplaced (1.992%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T14:47:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:12.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:12.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:12.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:13.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:13.232 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:13.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:13.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:13.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:13.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:13.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.604 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:13.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:47:13.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:13.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:13.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:13.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:13.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.886 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:13.886 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:13.916 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:13.916 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:13.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:13.949 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:13.949 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:13.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:13.971 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:13.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:13.993 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:13.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:14.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:14.019 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:14.038 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:14.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:14.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:14.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:14.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:14.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:14.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:14.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:14.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:14.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[103001]: pgmap v1153: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 33938/129390 objects degraded (26.229%); 2591/129390 objects misplaced (2.002%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:14.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 33938/129390 objects degraded (26.229%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:14.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1130176748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[107556]: pgmap v1153: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 33938/129390 objects degraded (26.229%); 2591/129390 objects misplaced (2.002%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:14.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 33938/129390 objects degraded (26.229%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:14.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1130176748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:14.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:14.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:14 smithi189 ceph-mon[90992]: pgmap v1153: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 33938/129390 objects degraded (26.229%); 2591/129390 objects misplaced (2.002%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 33938/129390 objects degraded (26.229%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1130176748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:14.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:14.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:14.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:15.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:15.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:15.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:15.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:15.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:15.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:15.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:15.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:15.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:15.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:15.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:15.938 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:16.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:16.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:16.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:16.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:16.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:16 smithi017 ceph-mon[107556]: pgmap v1154: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33880/128049 objects degraded (26.459%); 2532/128049 objects misplaced (1.977%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:16 smithi017 ceph-mon[103001]: pgmap v1154: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33880/128049 objects degraded (26.459%); 2532/128049 objects misplaced (1.977%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:16.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:16 smithi189 ceph-mon[90992]: pgmap v1154: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 33880/128049 objects degraded (26.459%); 2532/128049 objects misplaced (1.977%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:47:16.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:16.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.911 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:16.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:16.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:17.066 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:17.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:17.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:17.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:17.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:17.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:17.446 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.470 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:17.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:17.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:17.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:17] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-13T14:47:17.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:17.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:17.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:17.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:17.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:17.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:17.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:18.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:18.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:18.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:18.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:18.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:18.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:18.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:18.375 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:18.376 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:47:18.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:18.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:18.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:18.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:18.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:18 smithi017 ceph-mon[103001]: pgmap v1155: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33748/127119 objects degraded (26.548%); 2532/127119 objects misplaced (1.992%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:18.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:18 smithi017 ceph-mon[107556]: pgmap v1155: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33748/127119 objects degraded (26.548%); 2532/127119 objects misplaced (1.992%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:18.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:18.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:18 smithi189 ceph-mon[90992]: pgmap v1155: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33748/127119 objects degraded (26.548%); 2532/127119 objects misplaced (1.992%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:18.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:18.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:18.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:19.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:19.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:19.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.363 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:19.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:19.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:19.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:19.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:19 smithi017 ceph-mon[103001]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 33748/127119 objects degraded (26.548%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:19 smithi017 ceph-mon[107556]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 33748/127119 objects degraded (26.548%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:19.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:19 smithi189 ceph-mon[90992]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:19.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 33748/127119 objects degraded (26.548%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:19.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:19.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:19.967 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:19.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:20.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:20.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:20.343 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:47:20.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:20.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:20.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:20 smithi189 ceph-mon[90992]: pgmap v1156: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33748/125940 objects degraded (26.797%); 2532/125940 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:20.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:20.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:20 smithi017 ceph-mon[107556]: pgmap v1156: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33748/125940 objects degraded (26.797%); 2532/125940 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:20 smithi017 ceph-mon[103001]: pgmap v1156: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33748/125940 objects degraded (26.797%); 2532/125940 objects misplaced (2.010%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T14:47:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:20.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:20.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:20.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:20.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:20.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:21.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:21.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:21.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[103001]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:47:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:21 smithi017 ceph-mon[107556]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:47:21.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:21.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:21 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3189368685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:21 smithi189 ceph-mon[90992]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:47:21.768 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1155,"stamp":"2024-09-13T14:47:17.255347+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2779744950,"num_objects":42373,"num_object_clones":0,"num_object_copies":127119,"num_objects_missing_on_primary":1580,"num_objects_missing":1580,"num_objects_degraded":33748,"num_objects_misplaced":2532,"num_objects_unfound":0,"num_objects_dirty":42373,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":141280,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26023,"num_bytes_recovered":1641449629,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126365,"ondisk_log_size":126365,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12917968,"kb_used_data":7888144,"kb_used_omap":44,"kb_used_meta":5029715,"kb_avail":737043248,"statfs":{"total":767960285184,"available":754732285952,"internally_reserved":0,"allocated":8077459456,"data_stored":8062178095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5150428407},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2,0,1,1,2,1,7,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123731968,"num_objects":-1888,"num_object_clones":0,"num_object_copies":-5664,"num_objects_missing_on_primary":-116,"num_objects_missing":-116,"num_objects_degraded":-390,"num_objects_misplaced":-117,"num_objects_unfound":0,"num_objects_dirty":-1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1888,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":16646144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002815"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":606,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321136+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:47:07.321136+0000","last_peered":"2024-09-13T14:47:07.321136+0000","last_clean":"2024-09-13T14:47:07.321136+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:47:07.321136+0000","last_undegraded":"2024-09-13T14:47:07.321136+0000","last_fullsized":"2024-09-13T14:47:07.321136+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":359,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698600+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:46:51.698600+0000","last_peered":"2024-09-13T14:46:51.698600+0000","last_clean":"2024-09-13T14:46:51.698600+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:46:51.698600+0000","last_undegraded":"2024-09-13T14:46:51.698600+0000","last_fullsized":"2024-09-13T14:46:51.698600+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":288,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699831+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:46:51.699831+0000","last_peered":"2024-09-13T14:46:51.699831+0000","last_clean":"2024-09-13T14:46:51.699831+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:46:51.699831+0000","last_undegraded":"2024-09-13T14:46:51.699831+0000","last_fullsized":"2024-09-13T14:46:51.699831+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":591,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.325850+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:47:07.325850+0000","last_peered":"2024-09-13T14:47:07.325850+0000","last_clean":"2024-09-13T14:47:07.325850+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:47:07.325850+0000","last_undegraded":"2024-09-13T14:47:07.325850+0000","last_fullsized":"2024-09-13T14:47:07.325850+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":592,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326790+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:47:07.326790+0000","last_peered":"2024-09-13T14:47:07.326790+0000","last_clean":"2024-09-13T14:47:07.326790+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:47:07.326790+0000","last_undegraded":"2024-09-13T14:47:07.326790+0000","last_fullsized":"2024-09-13T14:47:07.326790+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":328,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698439+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:46:51.698439+0000","last_peered":"2024-09-13T14:46:51.698439+0000","last_clean":"2024-09-13T14:46:51.698439+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:46:51.698439+0000","last_undegraded":"2024-09-13T14:46:51.698439+0000","last_fullsized":"2024-09-13T14:46:51.698439+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":606,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320722+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:47:07.320722+0000","last_peered":"2024-09-13T14:47:07.320722+0000","last_clean":"2024-09-13T14:47:07.320722+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:47:07.320722+0000","last_undegraded":"2024-09-13T14:47:07.320722+0000","last_fullsized":"2024-09-13T14:47:07.320722+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":350,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702052+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:46:51.702052+0000","last_peered":"2024-09-13T14:46:51.702052+0000","last_clean":"2024-09-13T14:46:51.702052+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:46:51.702052+0000","last_undegraded":"2024-09-13T14:46:51.702052+0000","last_fullsized":"2024-09-13T14:46:51.702052+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"361'8742","reported_seq":18160,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:17.075339+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:47:17.075339+0000","last_peered":"2024-09-13T14:47:17.075339+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:47:17.075339+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8742,"log_dups_size":0,"ondisk_log_size":8742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4009,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8742,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3098,"num_bytes_recovered":203030528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1563}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":354,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698740+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:46:51.698740+0000","last_peered":"2024-09-13T14:46:51.698740+0000","last_clean":"2024-09-13T14:46:51.698740+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:46:51.698740+0000","last_undegraded":"2024-09-13T14:46:51.698740+0000","last_fullsized":"2024-09-13T14:46:51.698740+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":273,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698312+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:46:51.698312+0000","last_peered":"2024-09-13T14:46:51.698312+0000","last_clean":"2024-09-13T14:46:51.698312+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:46:51.698312+0000","last_undegraded":"2024-09-13T14:46:51.698312+0000","last_fullsized":"2024-09-13T14:46:51.698312+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"361'8795","reported_seq":18353,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:13.041682+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:47:13.041682+0000","last_peered":"2024-09-13T14:47:13.041682+0000","last_clean":"2024-09-13T14:47:13.041682+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:47:13.041682+0000","last_undegraded":"2024-09-13T14:47:13.041682+0000","last_fullsized":"2024-09-13T14:47:13.041682+0000","mapping_epoch":326,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8795,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":372,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698960+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:46:51.698960+0000","last_peered":"2024-09-13T14:46:51.698960+0000","last_clean":"2024-09-13T14:46:51.698960+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:46:51.698960+0000","last_undegraded":"2024-09-13T14:46:51.698960+0000","last_fullsized":"2024-09-13T14:46:51.698960+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700325+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:46:51.700325+0000","last_peered":"2024-09-13T14:46:51.700325+0000","last_clean":"2024-09-13T14:46:51.700325+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:46:51.700325+0000","last_undegraded":"2024-09-13T14:46:51.700325+0000","last_fullsized":"2024-09-13T14:46:51.700325+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"361'9091","reported_seq":9221,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:13.934334+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:47:13.934334+0000","last_peered":"2024-09-13T14:47:13.934334+0000","last_clean":"2024-09-13T14:47:13.934334+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:47:13.934334+0000","last_undegraded":"2024-09-13T14:47:13.934334+0000","last_fullsized":"2024-09-13T14:47:13.934334+0000","mapping_epoch":303,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8591,"log_dups_size":0,"ondisk_log_size":8591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177405952,"num_objects":2707,"num_object_clones":0,"num_object_copies":8121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9091,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":326,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699313+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:46:51.699313+0000","last_peered":"2024-09-13T14:46:51.699313+0000","last_clean":"2024-09-13T14:46:51.699313+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:46:51.699313+0000","last_undegraded":"2024-09-13T14:46:51.699313+0000","last_fullsized":"2024-09-13T14:46:51.699313+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":341,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702152+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:46:51.702152+0000","last_peered":"2024-09-13T14:46:51.702152+0000","last_clean":"2024-09-13T14:46:51.702152+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:46:51.702152+0000","last_undegraded":"2024-09-13T14:46:51.702152+0000","last_fullsized":"2024-09-13T14:46:51.702152+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"361'8878","reported_seq":9008,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:14.710547+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:47:14.710547+0000","last_peered":"2024-09-13T14:47:14.710547+0000","last_clean":"2024-09-13T14:47:14.710547+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:47:14.710547+0000","last_undegraded":"2024-09-13T14:47:14.710547+0000","last_fullsized":"2024-09-13T14:47:14.710547+0000","mapping_epoch":303,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174850048,"num_objects":2668,"num_object_clones":0,"num_object_copies":8004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8878,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":289,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698435+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:46:51.698435+0000","last_peered":"2024-09-13T14:46:51.698435+0000","last_clean":"2024-09-13T14:46:51.698435+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:46:51.698435+0000","last_undegraded":"2024-09-13T14:46:51.698435+0000","last_fullsized":"2024-09-13T14:46:51.698435+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":618,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.325343+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:47:07.325343+0000","last_peered":"2024-09-13T14:47:07.325343+0000","last_clean":"2024-09-13T14:47:07.325343+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:47:07.325343+0000","last_undegraded":"2024-09-13T14:47:07.325343+0000","last_fullsized":"2024-09-13T14:47:07.325343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"361'8825","reported_seq":8952,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:15.479047+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:47:15.479047+0000","last_peered":"2024-09-13T14:47:15.479047+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:47:15.479047+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7025,"log_dups_size":0,"ondisk_log_size":7025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172818432,"num_objects":2637,"num_object_clones":0,"num_object_copies":7911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8825,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2996}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":472,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698268+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:46:51.698268+0000","last_peered":"2024-09-13T14:46:51.698268+0000","last_clean":"2024-09-13T14:46:51.698268+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:46:51.698268+0000","last_undegraded":"2024-09-13T14:46:51.698268+0000","last_fullsized":"2024-09-13T14:46:51.698268+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320527+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:47:07.320527+0000","last_peered":"2024-09-13T14:47:07.320527+0000","last_clean":"2024-09-13T14:47:07.320527+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:47:07.320527+0000","last_undegraded":"2024-09-13T14:47:07.320527+0000","last_fullsized":"2024-09-13T14:47:07.320527+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700491+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:46:51.700491+0000","last_peered":"2024-09-13T14:46:51.700491+0000","last_clean":"2024-09-13T14:46:51.700491+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:46:51.700491+0000","last_undegraded":"2024-09-13T14:46:51.700491+0000","last_fullsized":"2024-09-13T14:46:51.700491+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"361'8530","reported_seq":8660,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:14.698951+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:47:14.698951+0000","last_peered":"2024-09-13T14:47:14.698951+0000","last_clean":"2024-09-13T14:47:14.698951+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:47:14.698951+0000","last_undegraded":"2024-09-13T14:47:14.698951+0000","last_fullsized":"2024-09-13T14:47:14.698951+0000","mapping_epoch":303,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168034304,"num_objects":2564,"num_object_clones":0,"num_object_copies":7692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8530,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":176,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698393+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:46:51.698393+0000","last_peered":"2024-09-13T14:46:51.698393+0000","last_clean":"2024-09-13T14:46:51.698393+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:46:51.698393+0000","last_undegraded":"2024-09-13T14:46:51.698393+0000","last_fullsized":"2024-09-13T14:46:51.698393+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":362,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702086+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:46:51.702086+0000","last_peered":"2024-09-13T14:46:51.702086+0000","last_clean":"2024-09-13T14:46:51.702086+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:46:51.702086+0000","last_undegraded":"2024-09-13T14:46:51.702086+0000","last_fullsized":"2024-09-13T14:46:51.702086+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":74,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700799+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:46:51.700799+0000","last_peered":"2024-09-13T14:46:51.700799+0000","last_clean":"2024-09-13T14:46:51.700799+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:46:51.700799+0000","last_undegraded":"2024-09-13T14:46:51.700799+0000","last_fullsized":"2024-09-13T14:46:51.700799+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"361'8776","reported_seq":17710,"reported_epoch":361,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:15.125742+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:47:15.125742+0000","last_peered":"2024-09-13T14:47:15.125742+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:47:15.125742+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172752896,"num_objects":2636,"num_object_clones":0,"num_object_copies":7908,"num_objects_missing_on_primary":1580,"num_objects_missing":1580,"num_objects_degraded":1955,"num_objects_misplaced":2532,"num_objects_unfound":0,"num_objects_dirty":2636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8776,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1580},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700201+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:46:51.700201+0000","last_peered":"2024-09-13T14:46:51.700201+0000","last_clean":"2024-09-13T14:46:51.700201+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:46:51.700201+0000","last_undegraded":"2024-09-13T14:46:51.700201+0000","last_fullsized":"2024-09-13T14:46:51.700201+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700826+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:46:51.700826+0000","last_peered":"2024-09-13T14:46:51.700826+0000","last_clean":"2024-09-13T14:46:51.700826+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:46:51.700826+0000","last_undegraded":"2024-09-13T14:46:51.700826+0000","last_fullsized":"2024-09-13T14:46:51.700826+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"361'8551","reported_seq":8681,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:14.851645+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:47:14.851645+0000","last_peered":"2024-09-13T14:47:14.851645+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:47:14.851645+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7551,"log_dups_size":0,"ondisk_log_size":7551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8551,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-2859}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":169,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698903+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:46:51.698903+0000","last_peered":"2024-09-13T14:46:51.698903+0000","last_clean":"2024-09-13T14:46:51.698903+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:46:51.698903+0000","last_undegraded":"2024-09-13T14:46:51.698903+0000","last_fullsized":"2024-09-13T14:46:51.698903+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":608,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328977+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:47:07.328977+0000","last_peered":"2024-09-13T14:47:07.328977+0000","last_clean":"2024-09-13T14:47:07.328977+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:47:07.328977+0000","last_undegraded":"2024-09-13T14:47:07.328977+0000","last_fullsized":"2024-09-13T14:47:07.328977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":570,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.331095+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:47:07.331095+0000","last_peered":"2024-09-13T14:47:07.331095+0000","last_clean":"2024-09-13T14:47:07.331095+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:47:07.331095+0000","last_undegraded":"2024-09-13T14:47:07.331095+0000","last_fullsized":"2024-09-13T14:47:07.331095+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"361'8590","reported_seq":8663,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:12.843082+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:47:12.843082+0000","last_peered":"2024-09-13T14:47:12.843082+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:47:12.843082+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'1000","ondisk_log_start":"306'1000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7590,"log_dups_size":0,"ondisk_log_size":7590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172359680,"num_objects":2630,"num_object_clones":0,"num_object_copies":7890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8590,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-2866}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":463,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699157+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:46:51.699157+0000","last_peered":"2024-09-13T14:46:51.699157+0000","last_clean":"2024-09-13T14:46:51.699157+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:46:51.699157+0000","last_undegraded":"2024-09-13T14:46:51.699157+0000","last_fullsized":"2024-09-13T14:46:51.699157+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":885,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698270+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:46:51.698270+0000","last_peered":"2024-09-13T14:46:51.698270+0000","last_clean":"2024-09-13T14:46:51.698270+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:46:51.698270+0000","last_undegraded":"2024-09-13T14:46:51.698270+0000","last_fullsized":"2024-09-13T14:46:51.698270+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":562,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319731+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:47:07.319731+0000","last_peered":"2024-09-13T14:47:07.319731+0000","last_clean":"2024-09-13T14:47:07.319731+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:47:07.319731+0000","last_undegraded":"2024-09-13T14:47:07.319731+0000","last_fullsized":"2024-09-13T14:47:07.319731+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702574+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:46:51.702574+0000","last_peered":"2024-09-13T14:46:51.702574+0000","last_clean":"2024-09-13T14:46:51.702574+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:46:51.702574+0000","last_undegraded":"2024-09-13T14:46:51.702574+0000","last_fullsized":"2024-09-13T14:46:51.702574+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"361'8905","reported_seq":9033,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:15.079835+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:47:15.079835+0000","last_peered":"2024-09-13T14:47:15.079835+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:47:15.079835+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173211648,"num_objects":2643,"num_object_clones":0,"num_object_copies":7929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8905,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3037}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":416,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328780+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:47:07.328780+0000","last_peered":"2024-09-13T14:47:07.328780+0000","last_clean":"2024-09-13T14:47:07.328780+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:47:07.328780+0000","last_undegraded":"2024-09-13T14:47:07.328780+0000","last_fullsized":"2024-09-13T14:47:07.328780+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":751,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699132+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:46:51.699132+0000","last_peered":"2024-09-13T14:46:51.699132+0000","last_clean":"2024-09-13T14:46:51.699132+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:46:51.699132+0000","last_undegraded":"2024-09-13T14:46:51.699132+0000","last_fullsized":"2024-09-13T14:46:51.699132+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":611,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326108+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:47:07.326108+0000","last_peered":"2024-09-13T14:47:07.326108+0000","last_clean":"2024-09-13T14:47:07.326108+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:47:07.326108+0000","last_undegraded":"2024-09-13T14:47:07.326108+0000","last_fullsized":"2024-09-13T14:47:07.326108+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":615,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326969+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:47:07.326969+0000","last_peered":"2024-09-13T14:47:07.326969+0000","last_clean":"2024-09-13T14:47:07.326969+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:47:07.326969+0000","last_undegraded":"2024-09-13T14:47:07.326969+0000","last_fullsized":"2024-09-13T14:47:07.326969+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"361'8924","reported_seq":18595,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:15.439149+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:47:15.439149+0000","last_peered":"2024-09-13T14:47:15.439149+0000","last_clean":"2024-09-13T14:47:15.439149+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:47:15.439149+0000","last_undegraded":"2024-09-13T14:47:15.439149+0000","last_fullsized":"2024-09-13T14:47:15.439149+0000","mapping_epoch":330,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179568640,"num_objects":2740,"num_object_clones":0,"num_object_copies":8220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8924,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":492,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698380+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:46:51.698380+0000","last_peered":"2024-09-13T14:46:51.698380+0000","last_clean":"2024-09-13T14:46:51.698380+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:46:51.698380+0000","last_undegraded":"2024-09-13T14:46:51.698380+0000","last_fullsized":"2024-09-13T14:46:51.698380+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":579,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320092+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:47:07.320092+0000","last_peered":"2024-09-13T14:47:07.320092+0000","last_clean":"2024-09-13T14:47:07.320092+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:47:07.320092+0000","last_undegraded":"2024-09-13T14:47:07.320092+0000","last_fullsized":"2024-09-13T14:47:07.320092+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":548,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328559+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:47:07.328559+0000","last_peered":"2024-09-13T14:47:07.328559+0000","last_clean":"2024-09-13T14:47:07.328559+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:47:07.328559+0000","last_undegraded":"2024-09-13T14:47:07.328559+0000","last_fullsized":"2024-09-13T14:47:07.328559+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":110,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698700+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:46:51.698700+0000","last_peered":"2024-09-13T14:46:51.698700+0000","last_clean":"2024-09-13T14:46:51.698700+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:46:51.698700+0000","last_undegraded":"2024-09-13T14:46:51.698700+0000","last_fullsized":"2024-09-13T14:46:51.698700+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"361'8578","reported_seq":8707,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:15.366691+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:47:15.366691+0000","last_peered":"2024-09-13T14:47:15.366691+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:47:15.366691+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8578,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-2857}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":169,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700891+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:46:51.700891+0000","last_peered":"2024-09-13T14:46:51.700891+0000","last_clean":"2024-09-13T14:46:51.700891+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:46:51.700891+0000","last_undegraded":"2024-09-13T14:46:51.700891+0000","last_fullsized":"2024-09-13T14:46:51.700891+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":976,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698977+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:46:51.698977+0000","last_peered":"2024-09-13T14:46:51.698977+0000","last_clean":"2024-09-13T14:46:51.698977+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:46:51.698977+0000","last_undegraded":"2024-09-13T14:46:51.698977+0000","last_fullsized":"2024-09-13T14:46:51.698977+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700639+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:46:51.700639+0000","last_peered":"2024-09-13T14:46:51.700639+0000","last_clean":"2024-09-13T14:46:51.700639+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:46:51.700639+0000","last_undegraded":"2024-09-13T14:46:51.700639+0000","last_fullsized":"2024-09-13T14:46:51.700639+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":569,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.330212+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:47:07.330212+0000","last_peered":"2024-09-13T14:47:07.330212+0000","last_clean":"2024-09-13T14:47:07.330212+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:47:07.330212+0000","last_undegraded":"2024-09-13T14:47:07.330212+0000","last_fullsized":"2024-09-13T14:47:07.330212+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"361'8720","reported_seq":18130,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:13.817907+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:47:13.817907+0000","last_peered":"2024-09-13T14:47:13.817907+0000","last_clean":"2024-09-13T14:47:13.817907+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:47:13.817907+0000","last_undegraded":"2024-09-13T14:47:13.817907+0000","last_fullsized":"2024-09-13T14:47:13.817907+0000","mapping_epoch":337,"log_start":"306'100","ondisk_log_start":"306'100","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8620,"log_dups_size":0,"ondisk_log_size":8620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173015040,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8720,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":360,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702595+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:46:51.702595+0000","last_peered":"2024-09-13T14:46:51.702595+0000","last_clean":"2024-09-13T14:46:51.702595+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:46:51.702595+0000","last_undegraded":"2024-09-13T14:46:51.702595+0000","last_fullsized":"2024-09-13T14:46:51.702595+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699700+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:46:51.699700+0000","last_peered":"2024-09-13T14:46:51.699700+0000","last_clean":"2024-09-13T14:46:51.699700+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:46:51.699700+0000","last_undegraded":"2024-09-13T14:46:51.699700+0000","last_fullsized":"2024-09-13T14:46:51.699700+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1148,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.330389+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:47:07.330389+0000","last_peered":"2024-09-13T14:47:07.330389+0000","last_clean":"2024-09-13T14:47:07.330389+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:47:07.330389+0000","last_undegraded":"2024-09-13T14:47:07.330389+0000","last_fullsized":"2024-09-13T14:47:07.330389+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":550,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.331669+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:47:07.331669+0000","last_peered":"2024-09-13T14:47:07.331669+0000","last_clean":"2024-09-13T14:47:07.331669+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:47:07.331669+0000","last_undegraded":"2024-09-13T14:47:07.331669+0000","last_fullsized":"2024-09-13T14:47:07.331669+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"361'8617","reported_seq":8747,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:15.536712+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:47:15.536712+0000","last_peered":"2024-09-13T14:47:15.536712+0000","last_clean":"2024-09-13T14:47:15.536712+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:47:15.536712+0000","last_undegraded":"2024-09-13T14:47:15.536712+0000","last_fullsized":"2024-09-13T14:47:15.536712+0000","mapping_epoch":303,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8617,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":360,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702493+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:46:51.702493+0000","last_peered":"2024-09-13T14:46:51.702493+0000","last_clean":"2024-09-13T14:46:51.702493+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:46:51.702493+0000","last_undegraded":"2024-09-13T14:46:51.702493+0000","last_fullsized":"2024-09-13T14:46:51.702493+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":559,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.330304+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:47:07.330304+0000","last_peered":"2024-09-13T14:47:07.330304+0000","last_clean":"2024-09-13T14:47:07.330304+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:47:07.330304+0000","last_undegraded":"2024-09-13T14:47:07.330304+0000","last_fullsized":"2024-09-13T14:47:07.330304+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":755,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699346+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:46:51.699346+0000","last_peered":"2024-09-13T14:46:51.699346+0000","last_clean":"2024-09-13T14:46:51.699346+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:46:51.699346+0000","last_undegraded":"2024-09-13T14:46:51.699346+0000","last_fullsized":"2024-09-13T14:46:51.699346+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":594,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.325277+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:47:07.325277+0000","last_peered":"2024-09-13T14:47:07.325277+0000","last_clean":"2024-09-13T14:47:07.325277+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:47:07.325277+0000","last_undegraded":"2024-09-13T14:47:07.325277+0000","last_fullsized":"2024-09-13T14:47:07.325277+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"361'8964","reported_seq":9094,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:13.973250+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:47:13.973250+0000","last_peered":"2024-09-13T14:47:13.973250+0000","last_clean":"2024-09-13T14:47:13.973250+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:47:13.973250+0000","last_undegraded":"2024-09-13T14:47:13.973250+0000","last_fullsized":"2024-09-13T14:47:13.973250+0000","mapping_epoch":303,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8664,"log_dups_size":0,"ondisk_log_size":8664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178651136,"num_objects":2726,"num_object_clones":0,"num_object_copies":8178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8964,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700683+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:46:51.700683+0000","last_peered":"2024-09-13T14:46:51.700683+0000","last_clean":"2024-09-13T14:46:51.700683+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:46:51.700683+0000","last_undegraded":"2024-09-13T14:46:51.700683+0000","last_fullsized":"2024-09-13T14:46:51.700683+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":594,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320977+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:47:07.320977+0000","last_peered":"2024-09-13T14:47:07.320977+0000","last_clean":"2024-09-13T14:47:07.320977+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:47:07.320977+0000","last_undegraded":"2024-09-13T14:47:07.320977+0000","last_fullsized":"2024-09-13T14:47:07.320977+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":934,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.330703+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:47:07.330703+0000","last_peered":"2024-09-13T14:47:07.330703+0000","last_clean":"2024-09-13T14:47:07.330703+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:47:07.330703+0000","last_undegraded":"2024-09-13T14:47:07.330703+0000","last_fullsized":"2024-09-13T14:47:07.330703+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":592,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328854+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:47:07.328854+0000","last_peered":"2024-09-13T14:47:07.328854+0000","last_clean":"2024-09-13T14:47:07.328854+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:47:07.328854+0000","last_undegraded":"2024-09-13T14:47:07.328854+0000","last_fullsized":"2024-09-13T14:47:07.328854+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"361'8847","reported_seq":20219,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:12.759499+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:47:12.759499+0000","last_peered":"2024-09-13T14:47:12.759499+0000","last_clean":"2024-09-13T14:47:12.759499+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:47:12.759499+0000","last_undegraded":"2024-09-13T14:47:12.759499+0000","last_fullsized":"2024-09-13T14:47:12.759499+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8847,"log_dups_size":0,"ondisk_log_size":8847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174719000,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8847,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":446,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320993+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:47:07.320993+0000","last_peered":"2024-09-13T14:47:07.320993+0000","last_clean":"2024-09-13T14:47:07.320993+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:47:07.320993+0000","last_undegraded":"2024-09-13T14:47:07.320993+0000","last_fullsized":"2024-09-13T14:47:07.320993+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":629,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326769+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:47:07.326769+0000","last_peered":"2024-09-13T14:47:07.326769+0000","last_clean":"2024-09-13T14:47:07.326769+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:47:07.326769+0000","last_undegraded":"2024-09-13T14:47:07.326769+0000","last_fullsized":"2024-09-13T14:47:07.326769+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":779,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702718+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:46:51.702718+0000","last_peered":"2024-09-13T14:46:51.702718+0000","last_clean":"2024-09-13T14:46:51.702718+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:46:51.702718+0000","last_undegraded":"2024-09-13T14:46:51.702718+0000","last_fullsized":"2024-09-13T14:46:51.702718+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700275+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:46:51.700275+0000","last_peered":"2024-09-13T14:46:51.700275+0000","last_clean":"2024-09-13T14:46:51.700275+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:46:51.700275+0000","last_undegraded":"2024-09-13T14:46:51.700275+0000","last_fullsized":"2024-09-13T14:46:51.700275+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700709+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:46:51.700709+0000","last_peered":"2024-09-13T14:46:51.700709+0000","last_clean":"2024-09-13T14:46:51.700709+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:46:51.700709+0000","last_undegraded":"2024-09-13T14:46:51.700709+0000","last_fullsized":"2024-09-13T14:46:51.700709+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":70,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700465+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:46:51.700465+0000","last_peered":"2024-09-13T14:46:51.700465+0000","last_clean":"2024-09-13T14:46:51.700465+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:46:51.700465+0000","last_undegraded":"2024-09-13T14:46:51.700465+0000","last_fullsized":"2024-09-13T14:46:51.700465+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":546,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.331477+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:47:07.331477+0000","last_peered":"2024-09-13T14:47:07.331477+0000","last_clean":"2024-09-13T14:47:07.331477+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:47:07.331477+0000","last_undegraded":"2024-09-13T14:47:07.331477+0000","last_fullsized":"2024-09-13T14:47:07.331477+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":360,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699270+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:46:51.699270+0000","last_peered":"2024-09-13T14:46:51.699270+0000","last_clean":"2024-09-13T14:46:51.699270+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:46:51.699270+0000","last_undegraded":"2024-09-13T14:46:51.699270+0000","last_fullsized":"2024-09-13T14:46:51.699270+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698707+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:46:51.698707+0000","last_peered":"2024-09-13T14:46:51.698707+0000","last_clean":"2024-09-13T14:46:51.698707+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:46:51.698707+0000","last_undegraded":"2024-09-13T14:46:51.698707+0000","last_fullsized":"2024-09-13T14:46:51.698707+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":550,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319840+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:47:07.319840+0000","last_peered":"2024-09-13T14:47:07.319840+0000","last_clean":"2024-09-13T14:47:07.319840+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:47:07.319840+0000","last_undegraded":"2024-09-13T14:47:07.319840+0000","last_fullsized":"2024-09-13T14:47:07.319840+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":602,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326364+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:47:07.326364+0000","last_peered":"2024-09-13T14:47:07.326364+0000","last_clean":"2024-09-13T14:47:07.326364+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:47:07.326364+0000","last_undegraded":"2024-09-13T14:47:07.326364+0000","last_fullsized":"2024-09-13T14:47:07.326364+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":606,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328807+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:47:07.328807+0000","last_peered":"2024-09-13T14:47:07.328807+0000","last_clean":"2024-09-13T14:47:07.328807+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:47:07.328807+0000","last_undegraded":"2024-09-13T14:47:07.328807+0000","last_fullsized":"2024-09-13T14:47:07.328807+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.702657+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:46:51.702657+0000","last_peered":"2024-09-13T14:46:51.702657+0000","last_clean":"2024-09-13T14:46:51.702657+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:46:51.702657+0000","last_undegraded":"2024-09-13T14:46:51.702657+0000","last_fullsized":"2024-09-13T14:46:51.702657+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":624,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.329192+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:47:07.329192+0000","last_peered":"2024-09-13T14:47:07.329192+0000","last_clean":"2024-09-13T14:47:07.329192+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:47:07.329192+0000","last_undegraded":"2024-09-13T14:47:07.329192+0000","last_fullsized":"2024-09-13T14:47:07.329192+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":594,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326873+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:47:07.326873+0000","last_peered":"2024-09-13T14:47:07.326873+0000","last_clean":"2024-09-13T14:47:07.326873+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:47:07.326873+0000","last_undegraded":"2024-09-13T14:47:07.326873+0000","last_fullsized":"2024-09-13T14:47:07.326873+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":571,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.328263+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:47:07.328263+0000","last_peered":"2024-09-13T14:47:07.328263+0000","last_clean":"2024-09-13T14:47:07.328263+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:47:07.328263+0000","last_undegraded":"2024-09-13T14:47:07.328263+0000","last_fullsized":"2024-09-13T14:47:07.328263+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":620,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319950+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:47:07.319950+0000","last_peered":"2024-09-13T14:47:07.319950+0000","last_clean":"2024-09-13T14:47:07.319950+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:47:07.319950+0000","last_undegraded":"2024-09-13T14:47:07.319950+0000","last_fullsized":"2024-09-13T14:47:07.319950+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":59,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.724111+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:46:51.724111+0000","last_peered":"2024-09-13T14:46:51.724111+0000","last_clean":"2024-09-13T14:46:51.724111+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:46:51.724111+0000","last_undegraded":"2024-09-13T14:46:51.724111+0000","last_fullsized":"2024-09-13T14:46:51.724111+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":517,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.330481+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:47:07.330481+0000","last_peered":"2024-09-13T14:47:07.330481+0000","last_clean":"2024-09-13T14:47:07.330481+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:47:07.330481+0000","last_undegraded":"2024-09-13T14:47:07.330481+0000","last_fullsized":"2024-09-13T14:47:07.330481+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":598,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.325192+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:47:07.325192+0000","last_peered":"2024-09-13T14:47:07.325192+0000","last_clean":"2024-09-13T14:47:07.325192+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:47:07.325192+0000","last_undegraded":"2024-09-13T14:47:07.325192+0000","last_fullsized":"2024-09-13T14:47:07.325192+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700630+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:46:51.700630+0000","last_peered":"2024-09-13T14:46:51.700630+0000","last_clean":"2024-09-13T14:46:51.700630+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:46:51.700630+0000","last_undegraded":"2024-09-13T14:46:51.700630+0000","last_fullsized":"2024-09-13T14:46:51.700630+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":326,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.698577+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:46:51.698577+0000","last_peered":"2024-09-13T14:46:51.698577+0000","last_clean":"2024-09-13T14:46:51.698577+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:46:51.698577+0000","last_undegraded":"2024-09-13T14:46:51.698577+0000","last_fullsized":"2024-09-13T14:46:51.698577+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.724035+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:46:51.724035+0000","last_peered":"2024-09-13T14:46:51.724035+0000","last_clean":"2024-09-13T14:46:51.724035+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:46:51.724035+0000","last_undegraded":"2024-09-13T14:46:51.724035+0000","last_fullsized":"2024-09-13T14:46:51.724035+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":292,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.699911+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:46:51.699911+0000","last_peered":"2024-09-13T14:46:51.699911+0000","last_clean":"2024-09-13T14:46:51.699911+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:46:51.699911+0000","last_undegraded":"2024-09-13T14:46:51.699911+0000","last_fullsized":"2024-09-13T14:46:51.699911+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":616,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320182+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:47:07.320182+0000","last_peered":"2024-09-13T14:47:07.320182+0000","last_clean":"2024-09-13T14:47:07.320182+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:47:07.320182+0000","last_undegraded":"2024-09-13T14:47:07.320182+0000","last_fullsized":"2024-09-13T14:47:07.320182+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":603,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320303+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:47:07.320303+0000","last_peered":"2024-09-13T14:47:07.320303+0000","last_clean":"2024-09-13T14:47:07.320303+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:47:07.320303+0000","last_undegraded":"2024-09-13T14:47:07.320303+0000","last_fullsized":"2024-09-13T14:47:07.320303+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.700139+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:46:51.700139+0000","last_peered":"2024-09-13T14:46:51.700139+0000","last_clean":"2024-09-13T14:46:51.700139+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:46:51.700139+0000","last_undegraded":"2024-09-13T14:46:51.700139+0000","last_fullsized":"2024-09-13T14:46:51.700139+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":608,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.326280+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:47:07.326280+0000","last_peered":"2024-09-13T14:47:07.326280+0000","last_clean":"2024-09-13T14:47:07.326280+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:47:07.326280+0000","last_undegraded":"2024-09-13T14:47:07.326280+0000","last_fullsized":"2024-09-13T14:47:07.326280+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2775056408,"num_objects":42345,"num_object_clones":0,"num_object_copies":127035,"num_objects_missing_on_primary":1580,"num_objects_missing":1580,"num_objects_degraded":33748,"num_objects_misplaced":2532,"num_objects_unfound":0,"num_objects_dirty":42345,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140333,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25966,"num_bytes_recovered":1641283584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8000974848,"data_stored":8000962632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124233,"ondisk_log_size":124233,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397933,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1775884,"kb_used_data":875716,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91969268,"statfs":{"total":95995035648,"available":94176530432,"internally_reserved":0,"allocated":896733184,"data_stored":894807458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594161,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1815800,"kb_used_data":1217584,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":91929352,"statfs":{"total":95995035648,"available":94135656448,"internally_reserved":0,"allocated":1246806016,"data_stored":1244900044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665615,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547064,"kb_used_data":222832,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93198088,"statfs":{"total":95995035648,"available":95434842112,"internally_reserved":0,"allocated":228179968,"data_stored":226270327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986617,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1547268,"kb_used_data":651580,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92197884,"statfs":{"total":95995035648,"available":94410633216,"internally_reserved":0,"allocated":667217920,"data_stored":665300292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2,0,1,1,2,1,7,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697177,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2148788,"kb_used_data":1390636,"kb_used_omap":4,"kb_used_meta":758139,"kb_avail":91596364,"statfs":{"total":95995035648,"available":93794676736,"internally_reserved":0,"allocated":1424011264,"data_stored":1422105776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":776334411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860704,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1930236,"kb_used_data":1304820,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91814916,"statfs":{"total":95995035648,"available":94018473984,"internally_reserved":0,"allocated":1336135680,"data_stored":1334222538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020375,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1513072,"kb_used_data":1195624,"kb_used_omap":9,"kb_used_meta":317430,"kb_avail":92232080,"statfs":{"total":95995035648,"available":94445649920,"internally_reserved":0,"allocated":1224318976,"data_stored":1222416961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":325049157},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187765,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1639856,"kb_used_data":1029352,"kb_used_omap":7,"kb_used_meta":610488,"kb_avail":92105296,"statfs":{"total":95995035648,"available":94315823104,"internally_reserved":0,"allocated":1054056448,"data_stored":1052154699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":625140043},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1042939904,"data_stored":1042939904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1210843136,"data_stored":1210843136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1326649344,"data_stored":1326645272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1413611520,"data_stored":1413611520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":661061632,"data_stored":661061632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":222629888,"data_stored":222625816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1239285760,"data_stored":1239285760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":883953664,"data_stored":883949592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:47:21.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:21.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:22.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:22.103 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:22.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:22.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:22.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:22 smithi017 ceph-mon[103001]: pgmap v1158: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 33688/124188 objects degraded (27.127%); 2472/124188 objects misplaced (1.991%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:22.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:22 smithi017 ceph-mon[107556]: pgmap v1158: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 33688/124188 objects degraded (27.127%); 2472/124188 objects misplaced (1.991%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:22.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:22.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:22.622 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:22.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:22.637 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:22.637 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:22.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:22 smithi189 ceph-mon[90992]: pgmap v1158: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 33688/124188 objects degraded (27.127%); 2472/124188 objects misplaced (1.991%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:22.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:22.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:23.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:23.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:23.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:23.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:23.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:23.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:23.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:23.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:47:23.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:23.745 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:23.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:23.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:23.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.000 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:24.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:24.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:47:24.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:24 smithi017 ceph-mon[103001]: pgmap v1159: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 33574/123417 objects degraded (27.204%); 2472/123417 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 33574/123417 objects degraded (27.204%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:24 smithi017 ceph-mon[107556]: pgmap v1159: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 33574/123417 objects degraded (27.204%); 2472/123417 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:24.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 33574/123417 objects degraded (27.204%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:24.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:24.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:24.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:24.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:24 smithi189 ceph-mon[90992]: pgmap v1159: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 199 op/s; 33574/123417 objects degraded (27.204%); 2472/123417 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:24.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 33574/123417 objects degraded (27.204%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:24.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:47:24.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:25.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:25.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:25.496 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:25.496 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:25.502 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:25.502 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:25.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:25.528 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:25.528 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:25.546 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:25.547 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:25.563 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:25.564 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:25.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:25.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:25.598 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:25.598 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:25.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:25.617 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:25.617 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:25.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:25.636 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:25.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:25.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:25.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:25.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:26.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:26.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:26.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:26.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:26.401 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:26.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:26.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:26.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:26.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:26 smithi017 ceph-mon[107556]: pgmap v1160: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 33511/122178 objects degraded (27.428%); 2411/122178 objects misplaced (1.973%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:26.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2467978424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:26.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:26 smithi017 ceph-mon[103001]: pgmap v1160: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 33511/122178 objects degraded (27.428%); 2411/122178 objects misplaced (1.973%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:26.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2467978424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:26.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:26.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:26 smithi189 ceph-mon[90992]: pgmap v1160: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 33511/122178 objects degraded (27.428%); 2411/122178 objects misplaced (1.973%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:26.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2467978424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:26.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:26.876 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:27.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:27.022 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:27.123 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:27.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:27.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:27.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:27.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:27] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T14:47:27.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:27.624 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:27.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:27.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:27.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:27.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:28.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:28.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:28.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:28.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:28.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:28.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:28.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:28.409 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:28.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:28.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:28.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:28.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:28.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:28 smithi017 ceph-mon[103001]: pgmap v1161: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33393/121377 objects degraded (27.512%); 2411/121377 objects misplaced (1.986%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:28.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:28 smithi017 ceph-mon[107556]: pgmap v1161: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33393/121377 objects degraded (27.512%); 2411/121377 objects misplaced (1.986%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:28.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:28 smithi189 ceph-mon[90992]: pgmap v1161: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 33393/121377 objects degraded (27.512%); 2411/121377 objects misplaced (1.986%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:28.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:28.785 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:28.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:28.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:28.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:29.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:29.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:29.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:29.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:29.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:29.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:29.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:29.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:29.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:29.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:29.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:29.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 33393/121377 objects degraded (27.512%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 33393/121377 objects degraded (27.512%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:29.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 33393/121377 objects degraded (27.512%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:29.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:29.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:29.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:29.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.044 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:30.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:30.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.366 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:47:30.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:30 smithi017 ceph-mon[103001]: pgmap v1162: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33393/120294 objects degraded (27.759%); 2411/120294 objects misplaced (2.004%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:30.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:30.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:30 smithi017 ceph-mon[107556]: pgmap v1162: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33393/120294 objects degraded (27.759%); 2411/120294 objects misplaced (2.004%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:30.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:30.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:30 smithi189 ceph-mon[90992]: pgmap v1162: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 33393/120294 objects degraded (27.759%); 2411/120294 objects misplaced (2.004%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:30.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:30.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:30.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:30.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:30.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:30.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:30.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:31.032 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:31.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:31.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:31.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:31.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[103001]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:47:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:31 smithi017 ceph-mon[107556]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:47:31.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.554 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:31.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:31.655 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:31.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:31.751 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:47:31.751 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:47:31.751 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:47:31.751 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:47:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:31 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3021752103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:31.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:31 smithi189 ceph-mon[90992]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:47:31.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:31.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:31.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:31.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:32.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:32.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:32.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:32.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:32.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:32.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:32.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:32.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:32.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:32 smithi017 ceph-mon[103001]: pgmap v1164: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 33333/118338 objects degraded (28.168%); 2348/118338 objects misplaced (1.984%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:32.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:32 smithi017 ceph-mon[107556]: pgmap v1164: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 33333/118338 objects degraded (28.168%); 2348/118338 objects misplaced (1.984%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:32.660 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:32.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:32.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:32 smithi189 ceph-mon[90992]: pgmap v1164: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 33333/118338 objects degraded (28.168%); 2348/118338 objects misplaced (1.984%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T14:47:32.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:32.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.807 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:32.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:32.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.964 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:32.965 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:47:33.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.218 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:33.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:33.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:33.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:33.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:33.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:33.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:33.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:47:33.784 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:33.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:34.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:34.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:34.188 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:34.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:34.410 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1164,"stamp":"2024-09-13T14:47:31.258893+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2587921078,"num_objects":39446,"num_object_clones":0,"num_object_copies":118338,"num_objects_missing_on_primary":1397,"num_objects_missing":1397,"num_objects_degraded":33333,"num_objects_misplaced":2348,"num_objects_unfound":0,"num_objects_dirty":39446,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":144207,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26323,"num_bytes_recovered":1661110429,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126592,"ondisk_log_size":126592,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12480400,"kb_used_data":7446416,"kb_used_omap":44,"kb_used_meta":5033875,"kb_avail":737480816,"statfs":{"total":767960285184,"available":755180355584,"internally_reserved":0,"allocated":7625129984,"data_stored":7609785847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5154688247},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,3,0,0,2,5,4,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-127795200,"num_objects":-1950,"num_object_clones":0,"num_object_copies":-5850,"num_objects_missing_on_primary":-123,"num_objects_missing":-123,"num_objects_degraded":-355,"num_objects_misplaced":-124,"num_objects_unfound":0,"num_objects_dirty":-1950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1950,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":240,"num_bytes_recovered":15728640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002459"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":611,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.390833+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:47:30.390833+0000","last_peered":"2024-09-13T14:47:30.390833+0000","last_clean":"2024-09-13T14:47:30.390833+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:47:30.390833+0000","last_undegraded":"2024-09-13T14:47:30.390833+0000","last_fullsized":"2024-09-13T14:47:30.390833+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":360,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363141+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:47:07.363141+0000","last_peered":"2024-09-13T14:47:07.363141+0000","last_clean":"2024-09-13T14:47:07.363141+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:47:07.363141+0000","last_undegraded":"2024-09-13T14:47:07.363141+0000","last_fullsized":"2024-09-13T14:47:07.363141+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":289,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320002+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:47:07.320002+0000","last_peered":"2024-09-13T14:47:07.320002+0000","last_clean":"2024-09-13T14:47:07.320002+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:47:07.320002+0000","last_undegraded":"2024-09-13T14:47:07.320002+0000","last_fullsized":"2024-09-13T14:47:07.320002+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":594,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362330+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:47:20.362330+0000","last_peered":"2024-09-13T14:47:20.362330+0000","last_clean":"2024-09-13T14:47:20.362330+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:47:20.362330+0000","last_undegraded":"2024-09-13T14:47:20.362330+0000","last_fullsized":"2024-09-13T14:47:20.362330+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":594,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.363002+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:47:20.363002+0000","last_peered":"2024-09-13T14:47:20.363002+0000","last_clean":"2024-09-13T14:47:20.363002+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:47:20.363002+0000","last_undegraded":"2024-09-13T14:47:20.363002+0000","last_fullsized":"2024-09-13T14:47:20.363002+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.362969+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:47:07.362969+0000","last_peered":"2024-09-13T14:47:07.362969+0000","last_clean":"2024-09-13T14:47:07.362969+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:47:07.362969+0000","last_undegraded":"2024-09-13T14:47:07.362969+0000","last_fullsized":"2024-09-13T14:47:07.362969+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":610,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.391501+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:47:30.391501+0000","last_peered":"2024-09-13T14:47:30.391501+0000","last_clean":"2024-09-13T14:47:30.391501+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:47:30.391501+0000","last_undegraded":"2024-09-13T14:47:30.391501+0000","last_fullsized":"2024-09-13T14:47:30.391501+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320565+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:47:07.320565+0000","last_peered":"2024-09-13T14:47:07.320565+0000","last_clean":"2024-09-13T14:47:07.320565+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:47:07.320565+0000","last_undegraded":"2024-09-13T14:47:07.320565+0000","last_fullsized":"2024-09-13T14:47:07.320565+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"362'8859","reported_seq":18627,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:26.908230+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:47:26.908230+0000","last_peered":"2024-09-13T14:47:26.908230+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:47:26.908230+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8859,"log_dups_size":0,"ondisk_log_size":8859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159318016,"num_objects":2431,"num_object_clones":0,"num_object_copies":7293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3777,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8859,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3214,"num_bytes_recovered":210632704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1447}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":355,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363443+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:47:07.363443+0000","last_peered":"2024-09-13T14:47:07.363443+0000","last_clean":"2024-09-13T14:47:07.363443+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:47:07.363443+0000","last_undegraded":"2024-09-13T14:47:07.363443+0000","last_fullsized":"2024-09-13T14:47:07.363443+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319723+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:47:07.319723+0000","last_peered":"2024-09-13T14:47:07.319723+0000","last_clean":"2024-09-13T14:47:07.319723+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:47:07.319723+0000","last_undegraded":"2024-09-13T14:47:07.319723+0000","last_fullsized":"2024-09-13T14:47:07.319723+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"362'8981","reported_seq":18540,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:27.828499+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:47:27.828499+0000","last_peered":"2024-09-13T14:47:27.828499+0000","last_clean":"2024-09-13T14:47:27.828499+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:47:27.828499+0000","last_undegraded":"2024-09-13T14:47:27.828499+0000","last_fullsized":"2024-09-13T14:47:27.828499+0000","mapping_epoch":326,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164298752,"num_objects":2507,"num_object_clones":0,"num_object_copies":7521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8981,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":373,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.364080+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:47:07.364080+0000","last_peered":"2024-09-13T14:47:07.364080+0000","last_clean":"2024-09-13T14:47:07.364080+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:47:07.364080+0000","last_undegraded":"2024-09-13T14:47:07.364080+0000","last_fullsized":"2024-09-13T14:47:07.364080+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332537+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:47:07.332537+0000","last_peered":"2024-09-13T14:47:07.332537+0000","last_clean":"2024-09-13T14:47:07.332537+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:47:07.332537+0000","last_undegraded":"2024-09-13T14:47:07.332537+0000","last_fullsized":"2024-09-13T14:47:07.332537+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"362'9281","reported_seq":9413,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:28.853006+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:47:28.853006+0000","last_peered":"2024-09-13T14:47:28.853006+0000","last_clean":"2024-09-13T14:47:28.853006+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:47:28.853006+0000","last_undegraded":"2024-09-13T14:47:28.853006+0000","last_fullsized":"2024-09-13T14:47:28.853006+0000","mapping_epoch":303,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8581,"log_dups_size":0,"ondisk_log_size":8581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9281,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":327,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363540+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:47:07.363540+0000","last_peered":"2024-09-13T14:47:07.363540+0000","last_clean":"2024-09-13T14:47:07.363540+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:47:07.363540+0000","last_undegraded":"2024-09-13T14:47:07.363540+0000","last_fullsized":"2024-09-13T14:47:07.363540+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":342,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320910+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:47:07.320910+0000","last_peered":"2024-09-13T14:47:07.320910+0000","last_clean":"2024-09-13T14:47:07.320910+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:47:07.320910+0000","last_undegraded":"2024-09-13T14:47:07.320910+0000","last_fullsized":"2024-09-13T14:47:07.320910+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"362'9050","reported_seq":9182,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:29.793024+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:47:29.793024+0000","last_peered":"2024-09-13T14:47:29.793024+0000","last_clean":"2024-09-13T14:47:29.793024+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:47:29.793024+0000","last_undegraded":"2024-09-13T14:47:29.793024+0000","last_fullsized":"2024-09-13T14:47:29.793024+0000","mapping_epoch":303,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9050,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":290,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320642+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:47:07.320642+0000","last_peered":"2024-09-13T14:47:07.320642+0000","last_clean":"2024-09-13T14:47:07.320642+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:47:07.320642+0000","last_undegraded":"2024-09-13T14:47:07.320642+0000","last_fullsized":"2024-09-13T14:47:07.320642+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":620,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362626+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:47:20.362626+0000","last_peered":"2024-09-13T14:47:20.362626+0000","last_clean":"2024-09-13T14:47:20.362626+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:47:20.362626+0000","last_undegraded":"2024-09-13T14:47:20.362626+0000","last_fullsized":"2024-09-13T14:47:20.362626+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"363'9019","reported_seq":9150,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:30.569045+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:47:30.569045+0000","last_peered":"2024-09-13T14:47:30.569045+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:47:30.569045+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'2000","ondisk_log_start":"307'2000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160104448,"num_objects":2443,"num_object_clones":0,"num_object_copies":7329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-3190}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":473,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.362770+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:47:07.362770+0000","last_peered":"2024-09-13T14:47:07.362770+0000","last_clean":"2024-09-13T14:47:07.362770+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:47:07.362770+0000","last_undegraded":"2024-09-13T14:47:07.362770+0000","last_fullsized":"2024-09-13T14:47:07.362770+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":620,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.390516+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:47:30.390516+0000","last_peered":"2024-09-13T14:47:30.390516+0000","last_clean":"2024-09-13T14:47:30.390516+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:47:30.390516+0000","last_undegraded":"2024-09-13T14:47:30.390516+0000","last_fullsized":"2024-09-13T14:47:30.390516+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332646+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:47:07.332646+0000","last_peered":"2024-09-13T14:47:07.332646+0000","last_clean":"2024-09-13T14:47:07.332646+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:47:07.332646+0000","last_undegraded":"2024-09-13T14:47:07.332646+0000","last_fullsized":"2024-09-13T14:47:07.332646+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"362'8704","reported_seq":8836,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:29.780236+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:47:29.780236+0000","last_peered":"2024-09-13T14:47:29.780236+0000","last_clean":"2024-09-13T14:47:29.780236+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:47:29.780236+0000","last_undegraded":"2024-09-13T14:47:29.780236+0000","last_fullsized":"2024-09-13T14:47:29.780236+0000","mapping_epoch":303,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7504,"log_dups_size":0,"ondisk_log_size":7504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156631040,"num_objects":2390,"num_object_clones":0,"num_object_copies":7170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":177,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319807+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:47:07.319807+0000","last_peered":"2024-09-13T14:47:07.319807+0000","last_clean":"2024-09-13T14:47:07.319807+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:47:07.319807+0000","last_undegraded":"2024-09-13T14:47:07.319807+0000","last_fullsized":"2024-09-13T14:47:07.319807+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":363,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320794+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:47:07.320794+0000","last_peered":"2024-09-13T14:47:07.320794+0000","last_clean":"2024-09-13T14:47:07.320794+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:47:07.320794+0000","last_undegraded":"2024-09-13T14:47:07.320794+0000","last_fullsized":"2024-09-13T14:47:07.320794+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332916+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:47:07.332916+0000","last_peered":"2024-09-13T14:47:07.332916+0000","last_clean":"2024-09-13T14:47:07.332916+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:47:07.332916+0000","last_undegraded":"2024-09-13T14:47:07.332916+0000","last_fullsized":"2024-09-13T14:47:07.332916+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"362'8961","reported_seq":18446,"reported_epoch":362,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:30.214373+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:47:30.214373+0000","last_peered":"2024-09-13T14:47:30.214373+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:47:30.214373+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":1397,"num_objects_missing":1397,"num_objects_degraded":1772,"num_objects_misplaced":2348,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3255,"num_bytes_recovered":213319680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1397},{"shards":"0,3,4","objects":951},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332370+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:47:07.332370+0000","last_peered":"2024-09-13T14:47:07.332370+0000","last_clean":"2024-09-13T14:47:07.332370+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:47:07.332370+0000","last_undegraded":"2024-09-13T14:47:07.332370+0000","last_fullsized":"2024-09-13T14:47:07.332370+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.333207+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:47:07.333207+0000","last_peered":"2024-09-13T14:47:07.333207+0000","last_clean":"2024-09-13T14:47:07.333207+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:47:07.333207+0000","last_undegraded":"2024-09-13T14:47:07.333207+0000","last_fullsized":"2024-09-13T14:47:07.333207+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"362'8738","reported_seq":8870,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:29.801178+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:47:29.801178+0000","last_peered":"2024-09-13T14:47:29.801178+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:47:29.801178+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158597120,"num_objects":2420,"num_object_clones":0,"num_object_copies":7260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8738,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-3046}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":170,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.324574+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:47:07.324574+0000","last_peered":"2024-09-13T14:47:07.324574+0000","last_clean":"2024-09-13T14:47:07.324574+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:47:07.324574+0000","last_undegraded":"2024-09-13T14:47:07.324574+0000","last_fullsized":"2024-09-13T14:47:07.324574+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":612,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.396621+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:47:30.396621+0000","last_peered":"2024-09-13T14:47:30.396621+0000","last_clean":"2024-09-13T14:47:30.396621+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:47:30.396621+0000","last_undegraded":"2024-09-13T14:47:30.396621+0000","last_fullsized":"2024-09-13T14:47:30.396621+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":572,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.349163+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:47:20.349163+0000","last_peered":"2024-09-13T14:47:20.349163+0000","last_clean":"2024-09-13T14:47:20.349163+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:47:20.349163+0000","last_undegraded":"2024-09-13T14:47:20.349163+0000","last_fullsized":"2024-09-13T14:47:20.349163+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"362'8781","reported_seq":8855,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:27.921752+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:47:27.921752+0000","last_peered":"2024-09-13T14:47:27.921752+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:47:27.921752+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159842304,"num_objects":2439,"num_object_clones":0,"num_object_copies":7317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8781,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-3057}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":464,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363027+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:47:07.363027+0000","last_peered":"2024-09-13T14:47:07.363027+0000","last_clean":"2024-09-13T14:47:07.363027+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:47:07.363027+0000","last_undegraded":"2024-09-13T14:47:07.363027+0000","last_fullsized":"2024-09-13T14:47:07.363027+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":886,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320190+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:47:07.320190+0000","last_peered":"2024-09-13T14:47:07.320190+0000","last_clean":"2024-09-13T14:47:07.320190+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:47:07.320190+0000","last_undegraded":"2024-09-13T14:47:07.320190+0000","last_fullsized":"2024-09-13T14:47:07.320190+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":566,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.389909+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:47:30.389909+0000","last_peered":"2024-09-13T14:47:30.389909+0000","last_clean":"2024-09-13T14:47:30.389909+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:47:30.389909+0000","last_undegraded":"2024-09-13T14:47:30.389909+0000","last_fullsized":"2024-09-13T14:47:30.389909+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":328,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321467+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:47:07.321467+0000","last_peered":"2024-09-13T14:47:07.321467+0000","last_clean":"2024-09-13T14:47:07.321467+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:47:07.321467+0000","last_undegraded":"2024-09-13T14:47:07.321467+0000","last_fullsized":"2024-09-13T14:47:07.321467+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"362'9089","reported_seq":9219,"reported_epoch":362,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:29.684738+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:47:29.684738+0000","last_peered":"2024-09-13T14:47:29.684738+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:47:29.684738+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161153024,"num_objects":2459,"num_object_clones":0,"num_object_copies":7377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9089,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3221}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":418,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.345577+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:47:20.345577+0000","last_peered":"2024-09-13T14:47:20.345577+0000","last_clean":"2024-09-13T14:47:20.345577+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:47:20.345577+0000","last_undegraded":"2024-09-13T14:47:20.345577+0000","last_fullsized":"2024-09-13T14:47:20.345577+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":752,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.362996+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:47:07.362996+0000","last_peered":"2024-09-13T14:47:07.362996+0000","last_clean":"2024-09-13T14:47:07.362996+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:47:07.362996+0000","last_undegraded":"2024-09-13T14:47:07.362996+0000","last_fullsized":"2024-09-13T14:47:07.362996+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":613,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362408+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:47:20.362408+0000","last_peered":"2024-09-13T14:47:20.362408+0000","last_clean":"2024-09-13T14:47:20.362408+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:47:20.362408+0000","last_undegraded":"2024-09-13T14:47:20.362408+0000","last_fullsized":"2024-09-13T14:47:20.362408+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":617,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.363090+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:47:20.363090+0000","last_peered":"2024-09-13T14:47:20.363090+0000","last_clean":"2024-09-13T14:47:20.363090+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:47:20.363090+0000","last_undegraded":"2024-09-13T14:47:20.363090+0000","last_fullsized":"2024-09-13T14:47:20.363090+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"363'9113","reported_seq":18788,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.492766+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:47:30.492766+0000","last_peered":"2024-09-13T14:47:30.492766+0000","last_clean":"2024-09-13T14:47:30.492766+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:47:30.492766+0000","last_undegraded":"2024-09-13T14:47:30.492766+0000","last_fullsized":"2024-09-13T14:47:30.492766+0000","mapping_epoch":330,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167182336,"num_objects":2551,"num_object_clones":0,"num_object_copies":7653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9113,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":493,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.362919+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:47:07.362919+0000","last_peered":"2024-09-13T14:47:07.362919+0000","last_clean":"2024-09-13T14:47:07.362919+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:47:07.362919+0000","last_undegraded":"2024-09-13T14:47:07.362919+0000","last_fullsized":"2024-09-13T14:47:07.362919+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":583,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.390182+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:47:30.390182+0000","last_peered":"2024-09-13T14:47:30.390182+0000","last_clean":"2024-09-13T14:47:30.390182+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:47:30.390182+0000","last_undegraded":"2024-09-13T14:47:30.390182+0000","last_fullsized":"2024-09-13T14:47:30.390182+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":550,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.345139+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:47:20.345139+0000","last_peered":"2024-09-13T14:47:20.345139+0000","last_clean":"2024-09-13T14:47:20.345139+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:47:20.345139+0000","last_undegraded":"2024-09-13T14:47:20.345139+0000","last_fullsized":"2024-09-13T14:47:20.345139+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":111,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320252+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:47:07.320252+0000","last_peered":"2024-09-13T14:47:07.320252+0000","last_clean":"2024-09-13T14:47:07.320252+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:47:07.320252+0000","last_undegraded":"2024-09-13T14:47:07.320252+0000","last_fullsized":"2024-09-13T14:47:07.320252+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"363'8779","reported_seq":8912,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:30.417028+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:47:30.417028+0000","last_peered":"2024-09-13T14:47:30.417028+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:47:30.417028+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8779,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-3058}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":170,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319918+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:47:07.319918+0000","last_peered":"2024-09-13T14:47:07.319918+0000","last_clean":"2024-09-13T14:47:07.319918+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:47:07.319918+0000","last_undegraded":"2024-09-13T14:47:07.319918+0000","last_fullsized":"2024-09-13T14:47:07.319918+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":977,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.324573+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:47:07.324573+0000","last_peered":"2024-09-13T14:47:07.324573+0000","last_clean":"2024-09-13T14:47:07.324573+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:47:07.324573+0000","last_undegraded":"2024-09-13T14:47:07.324573+0000","last_fullsized":"2024-09-13T14:47:07.324573+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332758+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:47:07.332758+0000","last_peered":"2024-09-13T14:47:07.332758+0000","last_clean":"2024-09-13T14:47:07.332758+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:47:07.332758+0000","last_undegraded":"2024-09-13T14:47:07.332758+0000","last_fullsized":"2024-09-13T14:47:07.332758+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":571,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347226+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:47:20.347226+0000","last_peered":"2024-09-13T14:47:20.347226+0000","last_clean":"2024-09-13T14:47:20.347226+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:47:20.347226+0000","last_undegraded":"2024-09-13T14:47:20.347226+0000","last_fullsized":"2024-09-13T14:47:20.347226+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"362'8909","reported_seq":18321,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:28.998910+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:47:28.998910+0000","last_peered":"2024-09-13T14:47:28.998910+0000","last_clean":"2024-09-13T14:47:28.998910+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:47:28.998910+0000","last_undegraded":"2024-09-13T14:47:28.998910+0000","last_fullsized":"2024-09-13T14:47:28.998910+0000","mapping_epoch":337,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8609,"log_dups_size":0,"ondisk_log_size":8609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8909,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321540+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:47:07.321540+0000","last_peered":"2024-09-13T14:47:07.321540+0000","last_clean":"2024-09-13T14:47:07.321540+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:47:07.321540+0000","last_undegraded":"2024-09-13T14:47:07.321540+0000","last_fullsized":"2024-09-13T14:47:07.321540+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":307,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.319895+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:47:07.319895+0000","last_peered":"2024-09-13T14:47:07.319895+0000","last_clean":"2024-09-13T14:47:07.319895+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:47:07.319895+0000","last_undegraded":"2024-09-13T14:47:07.319895+0000","last_fullsized":"2024-09-13T14:47:07.319895+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1150,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347820+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:47:20.347820+0000","last_peered":"2024-09-13T14:47:20.347820+0000","last_clean":"2024-09-13T14:47:20.347820+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:47:20.347820+0000","last_undegraded":"2024-09-13T14:47:20.347820+0000","last_fullsized":"2024-09-13T14:47:20.347820+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":552,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347484+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:47:20.347484+0000","last_peered":"2024-09-13T14:47:20.347484+0000","last_clean":"2024-09-13T14:47:20.347484+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:47:20.347484+0000","last_undegraded":"2024-09-13T14:47:20.347484+0000","last_fullsized":"2024-09-13T14:47:20.347484+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"363'8804","reported_seq":8938,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.621424+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:47:30.621424+0000","last_peered":"2024-09-13T14:47:30.621424+0000","last_clean":"2024-09-13T14:47:30.621424+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:47:30.621424+0000","last_undegraded":"2024-09-13T14:47:30.621424+0000","last_fullsized":"2024-09-13T14:47:30.621424+0000","mapping_epoch":303,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8804,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321308+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:47:07.321308+0000","last_peered":"2024-09-13T14:47:07.321308+0000","last_clean":"2024-09-13T14:47:07.321308+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:47:07.321308+0000","last_undegraded":"2024-09-13T14:47:07.321308+0000","last_fullsized":"2024-09-13T14:47:07.321308+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":561,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347513+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:47:20.347513+0000","last_peered":"2024-09-13T14:47:20.347513+0000","last_clean":"2024-09-13T14:47:20.347513+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:47:20.347513+0000","last_undegraded":"2024-09-13T14:47:20.347513+0000","last_fullsized":"2024-09-13T14:47:20.347513+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":756,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363610+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:47:07.363610+0000","last_peered":"2024-09-13T14:47:07.363610+0000","last_clean":"2024-09-13T14:47:07.363610+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:47:07.363610+0000","last_undegraded":"2024-09-13T14:47:07.363610+0000","last_fullsized":"2024-09-13T14:47:07.363610+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":596,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362531+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:47:20.362531+0000","last_peered":"2024-09-13T14:47:20.362531+0000","last_clean":"2024-09-13T14:47:20.362531+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:47:20.362531+0000","last_undegraded":"2024-09-13T14:47:20.362531+0000","last_fullsized":"2024-09-13T14:47:20.362531+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"362'9160","reported_seq":9292,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:28.832266+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:47:28.832266+0000","last_peered":"2024-09-13T14:47:28.832266+0000","last_clean":"2024-09-13T14:47:28.832266+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:47:28.832266+0000","last_undegraded":"2024-09-13T14:47:28.832266+0000","last_fullsized":"2024-09-13T14:47:28.832266+0000","mapping_epoch":303,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8660,"log_dups_size":0,"ondisk_log_size":8660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9160,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332763+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:47:07.332763+0000","last_peered":"2024-09-13T14:47:07.332763+0000","last_clean":"2024-09-13T14:47:07.332763+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:47:07.332763+0000","last_undegraded":"2024-09-13T14:47:07.332763+0000","last_fullsized":"2024-09-13T14:47:07.332763+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":598,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.391659+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:47:30.391659+0000","last_peered":"2024-09-13T14:47:30.391659+0000","last_clean":"2024-09-13T14:47:30.391659+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:47:30.391659+0000","last_undegraded":"2024-09-13T14:47:30.391659+0000","last_fullsized":"2024-09-13T14:47:30.391659+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":936,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347672+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:47:20.347672+0000","last_peered":"2024-09-13T14:47:20.347672+0000","last_clean":"2024-09-13T14:47:20.347672+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:47:20.347672+0000","last_undegraded":"2024-09-13T14:47:20.347672+0000","last_fullsized":"2024-09-13T14:47:20.347672+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":596,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.396520+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:47:30.396520+0000","last_peered":"2024-09-13T14:47:30.396520+0000","last_clean":"2024-09-13T14:47:30.396520+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:47:30.396520+0000","last_undegraded":"2024-09-13T14:47:30.396520+0000","last_fullsized":"2024-09-13T14:47:30.396520+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"362'9032","reported_seq":20405,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:27.667274+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:47:27.667274+0000","last_peered":"2024-09-13T14:47:27.667274+0000","last_clean":"2024-09-13T14:47:27.667274+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:47:27.667274+0000","last_undegraded":"2024-09-13T14:47:27.667274+0000","last_fullsized":"2024-09-13T14:47:27.667274+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9032,"log_dups_size":0,"ondisk_log_size":9032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594840,"num_objects":2482,"num_object_clones":0,"num_object_copies":7446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2482,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9032,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":450,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.390694+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:47:30.390694+0000","last_peered":"2024-09-13T14:47:30.390694+0000","last_clean":"2024-09-13T14:47:30.390694+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:47:30.390694+0000","last_undegraded":"2024-09-13T14:47:30.390694+0000","last_fullsized":"2024-09-13T14:47:30.390694+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":631,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362896+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:47:20.362896+0000","last_peered":"2024-09-13T14:47:20.362896+0000","last_clean":"2024-09-13T14:47:20.362896+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:47:20.362896+0000","last_undegraded":"2024-09-13T14:47:20.362896+0000","last_fullsized":"2024-09-13T14:47:20.362896+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":780,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321610+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:47:07.321610+0000","last_peered":"2024-09-13T14:47:07.321610+0000","last_clean":"2024-09-13T14:47:07.321610+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:47:07.321610+0000","last_undegraded":"2024-09-13T14:47:07.321610+0000","last_fullsized":"2024-09-13T14:47:07.321610+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332447+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:47:07.332447+0000","last_peered":"2024-09-13T14:47:07.332447+0000","last_clean":"2024-09-13T14:47:07.332447+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:47:07.332447+0000","last_undegraded":"2024-09-13T14:47:07.332447+0000","last_fullsized":"2024-09-13T14:47:07.332447+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332834+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:47:07.332834+0000","last_peered":"2024-09-13T14:47:07.332834+0000","last_clean":"2024-09-13T14:47:07.332834+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:47:07.332834+0000","last_undegraded":"2024-09-13T14:47:07.332834+0000","last_fullsized":"2024-09-13T14:47:07.332834+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":71,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332617+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:47:07.332617+0000","last_peered":"2024-09-13T14:47:07.332617+0000","last_clean":"2024-09-13T14:47:07.332617+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:47:07.332617+0000","last_undegraded":"2024-09-13T14:47:07.332617+0000","last_fullsized":"2024-09-13T14:47:07.332617+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":548,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.346908+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:47:20.346908+0000","last_peered":"2024-09-13T14:47:20.346908+0000","last_clean":"2024-09-13T14:47:20.346908+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:47:20.346908+0000","last_undegraded":"2024-09-13T14:47:20.346908+0000","last_fullsized":"2024-09-13T14:47:20.346908+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":361,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.363421+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:47:07.363421+0000","last_peered":"2024-09-13T14:47:07.363421+0000","last_clean":"2024-09-13T14:47:07.363421+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:47:07.363421+0000","last_undegraded":"2024-09-13T14:47:07.363421+0000","last_fullsized":"2024-09-13T14:47:07.363421+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":327,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320166+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:47:07.320166+0000","last_peered":"2024-09-13T14:47:07.320166+0000","last_clean":"2024-09-13T14:47:07.320166+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:47:07.320166+0000","last_undegraded":"2024-09-13T14:47:07.320166+0000","last_fullsized":"2024-09-13T14:47:07.320166+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":555,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.389913+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:47:30.389913+0000","last_peered":"2024-09-13T14:47:30.389913+0000","last_clean":"2024-09-13T14:47:30.389913+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:47:30.389913+0000","last_undegraded":"2024-09-13T14:47:30.389913+0000","last_fullsized":"2024-09-13T14:47:30.389913+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":604,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.363045+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:47:20.363045+0000","last_peered":"2024-09-13T14:47:20.363045+0000","last_clean":"2024-09-13T14:47:20.363045+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:47:20.363045+0000","last_undegraded":"2024-09-13T14:47:20.363045+0000","last_fullsized":"2024-09-13T14:47:20.363045+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":611,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.396456+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:47:30.396456+0000","last_peered":"2024-09-13T14:47:30.396456+0000","last_clean":"2024-09-13T14:47:30.396456+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:47:30.396456+0000","last_undegraded":"2024-09-13T14:47:30.396456+0000","last_fullsized":"2024-09-13T14:47:30.396456+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":359,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.321535+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:47:07.321535+0000","last_peered":"2024-09-13T14:47:07.321535+0000","last_clean":"2024-09-13T14:47:07.321535+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:47:07.321535+0000","last_undegraded":"2024-09-13T14:47:07.321535+0000","last_fullsized":"2024-09-13T14:47:07.321535+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":628,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.396567+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:47:30.396567+0000","last_peered":"2024-09-13T14:47:30.396567+0000","last_clean":"2024-09-13T14:47:30.396567+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:47:30.396567+0000","last_undegraded":"2024-09-13T14:47:30.396567+0000","last_fullsized":"2024-09-13T14:47:30.396567+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":596,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.363044+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:47:20.363044+0000","last_peered":"2024-09-13T14:47:20.363044+0000","last_clean":"2024-09-13T14:47:20.363044+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:47:20.363044+0000","last_undegraded":"2024-09-13T14:47:20.363044+0000","last_fullsized":"2024-09-13T14:47:20.363044+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":573,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.345105+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:47:20.345105+0000","last_peered":"2024-09-13T14:47:20.345105+0000","last_clean":"2024-09-13T14:47:20.345105+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:47:20.345105+0000","last_undegraded":"2024-09-13T14:47:20.345105+0000","last_fullsized":"2024-09-13T14:47:20.345105+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":624,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.391087+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:47:30.391087+0000","last_peered":"2024-09-13T14:47:30.391087+0000","last_clean":"2024-09-13T14:47:30.391087+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:47:30.391087+0000","last_undegraded":"2024-09-13T14:47:30.391087+0000","last_fullsized":"2024-09-13T14:47:30.391087+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":60,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.338895+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:47:07.338895+0000","last_peered":"2024-09-13T14:47:07.338895+0000","last_clean":"2024-09-13T14:47:07.338895+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:47:07.338895+0000","last_undegraded":"2024-09-13T14:47:07.338895+0000","last_fullsized":"2024-09-13T14:47:07.338895+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":519,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.347925+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:47:20.347925+0000","last_peered":"2024-09-13T14:47:20.347925+0000","last_clean":"2024-09-13T14:47:20.347925+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:47:20.347925+0000","last_undegraded":"2024-09-13T14:47:20.347925+0000","last_fullsized":"2024-09-13T14:47:20.347925+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":600,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362387+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:47:20.362387+0000","last_peered":"2024-09-13T14:47:20.362387+0000","last_clean":"2024-09-13T14:47:20.362387+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:47:20.362387+0000","last_undegraded":"2024-09-13T14:47:20.362387+0000","last_fullsized":"2024-09-13T14:47:20.362387+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332683+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:47:07.332683+0000","last_peered":"2024-09-13T14:47:07.332683+0000","last_clean":"2024-09-13T14:47:07.332683+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:47:07.332683+0000","last_undegraded":"2024-09-13T14:47:07.332683+0000","last_fullsized":"2024-09-13T14:47:07.332683+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":327,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320704+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:47:07.320704+0000","last_peered":"2024-09-13T14:47:07.320704+0000","last_clean":"2024-09-13T14:47:07.320704+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:47:07.320704+0000","last_undegraded":"2024-09-13T14:47:07.320704+0000","last_fullsized":"2024-09-13T14:47:07.320704+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":60,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.338802+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:47:07.338802+0000","last_peered":"2024-09-13T14:47:07.338802+0000","last_clean":"2024-09-13T14:47:07.338802+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:47:07.338802+0000","last_undegraded":"2024-09-13T14:47:07.338802+0000","last_fullsized":"2024-09-13T14:47:07.338802+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":293,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.320074+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:47:07.320074+0000","last_peered":"2024-09-13T14:47:07.320074+0000","last_clean":"2024-09-13T14:47:07.320074+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:47:07.320074+0000","last_undegraded":"2024-09-13T14:47:07.320074+0000","last_fullsized":"2024-09-13T14:47:07.320074+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":620,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.390284+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:47:30.390284+0000","last_peered":"2024-09-13T14:47:30.390284+0000","last_clean":"2024-09-13T14:47:30.390284+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:47:30.390284+0000","last_undegraded":"2024-09-13T14:47:30.390284+0000","last_fullsized":"2024-09-13T14:47:30.390284+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":607,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.391476+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:47:30.391476+0000","last_peered":"2024-09-13T14:47:30.391476+0000","last_clean":"2024-09-13T14:47:30.391476+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:47:30.391476+0000","last_undegraded":"2024-09-13T14:47:30.391476+0000","last_fullsized":"2024-09-13T14:47:30.391476+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-09-13T14:47:07.332362+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:47:07.332362+0000","last_peered":"2024-09-13T14:47:07.332362+0000","last_clean":"2024-09-13T14:47:07.332362+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:47:07.332362+0000","last_undegraded":"2024-09-13T14:47:07.332362+0000","last_fullsized":"2024-09-13T14:47:07.332362+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":610,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-09-13T14:47:20.362832+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:47:20.362832+0000","last_peered":"2024-09-13T14:47:20.362832+0000","last_clean":"2024-09-13T14:47:20.362832+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:47:20.362832+0000","last_undegraded":"2024-09-13T14:47:20.362832+0000","last_fullsized":"2024-09-13T14:47:20.362832+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2583232536,"num_objects":39418,"num_object_clones":0,"num_object_copies":118254,"num_objects_missing_on_primary":1397,"num_objects_missing":1397,"num_objects_degraded":33333,"num_objects_misplaced":2348,"num_objects_unfound":0,"num_objects_dirty":39418,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143260,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26266,"num_bytes_recovered":1660944384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7550545920,"data_stored":7550533704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124460,"ondisk_log_size":124460,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397936,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702108,"kb_used_data":814420,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":92043044,"statfs":{"total":95995035648,"available":94252077056,"internally_reserved":0,"allocated":833966080,"data_stored":832032507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594164,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761352,"kb_used_data":1158976,"kb_used_omap":1,"kb_used_meta":602366,"kb_avail":91983800,"statfs":{"total":95995035648,"available":94191411200,"internally_reserved":0,"allocated":1186791424,"data_stored":1184877605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616823128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665618,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543816,"kb_used_data":211264,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93201336,"statfs":{"total":95995035648,"available":95438168064,"internally_reserved":0,"allocated":216334336,"data_stored":214416848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986620,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514580,"kb_used_data":614732,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92230572,"statfs":{"total":95995035648,"available":94444105728,"internally_reserved":0,"allocated":629485568,"data_stored":627560093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,0,0,2,5,4,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697180,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2062724,"kb_used_data":1317052,"kb_used_omap":4,"kb_used_meta":745659,"kb_avail":91682428,"statfs":{"total":95995035648,"available":93882806272,"internally_reserved":0,"allocated":1348661248,"data_stored":1346747913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":763554891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860706,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888268,"kb_used_data":1262852,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":91856884,"statfs":{"total":95995035648,"available":94061449216,"internally_reserved":0,"allocated":1293160448,"data_stored":1291239459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020378,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1437376,"kb_used_data":1111608,"kb_used_omap":9,"kb_used_meta":325750,"kb_avail":92307776,"statfs":{"total":95995035648,"available":94523162624,"internally_reserved":0,"allocated":1138286592,"data_stored":1136376730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":333568837},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187768,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1570176,"kb_used_data":955512,"kb_used_omap":7,"kb_used_meta":614648,"kb_avail":92174976,"statfs":{"total":95995035648,"available":94387175424,"internally_reserved":0,"allocated":978444288,"data_stored":976534692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":629399883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":968753152,"data_stored":968753152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1126760448,"data_stored":1126760448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1280970752,"data_stored":1280966680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1339424768,"data_stored":1339424768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":623771648,"data_stored":623771648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":210505728,"data_stored":210501656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1178927104,"data_stored":1178927104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":821432320,"data_stored":821428248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:47:34.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:47:34.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:34.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:47:34.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:34.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:34.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:34.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:34.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[107556]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[107556]: pgmap v1165: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 33193/117462 objects degraded (28.259%); 2348/117462 objects misplaced (1.999%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 33193/117462 objects degraded (28.259%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[103001]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[103001]: pgmap v1165: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 33193/117462 objects degraded (28.259%); 2348/117462 objects misplaced (1.999%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 33193/117462 objects degraded (28.259%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:34 smithi189 ceph-mon[90992]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:34 smithi189 ceph-mon[90992]: pgmap v1165: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 33193/117462 objects degraded (28.259%); 2348/117462 objects misplaced (1.999%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:47:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 33193/117462 objects degraded (28.259%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:34.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:34.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:34.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:35.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:35.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:35.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:35.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.362 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:35.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:35.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:35.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:35.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:35.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:35.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:35.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.186 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.209 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:36.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:36.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:36.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:36.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:36.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:36.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:36.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:36 smithi017 ceph-mon[103001]: pgmap v1166: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 33193/116133 objects degraded (28.582%); 2348/116133 objects misplaced (2.022%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:47:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:36 smithi017 ceph-mon[107556]: pgmap v1166: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 33193/116133 objects degraded (28.582%); 2348/116133 objects misplaced (2.022%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:47:36.705 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:36.709 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:36 smithi189 ceph-mon[90992]: pgmap v1166: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 33193/116133 objects degraded (28.582%); 2348/116133 objects misplaced (2.022%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:47:36.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:36.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:36.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:36.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:36.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:37.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:37.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:37.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:37.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:37.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:37.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:37.560 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:37.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:37.669 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:37] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T14:47:37.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:37.691 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:47:37.691 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:47:37.691 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:47:37.691 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:47:37.691 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:47:37.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:37.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:37.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:37.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:37.955 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:38.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:38.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:38.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.205 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:38.319 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:38.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:38.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:47:38.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:38.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:38.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:38.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:38 smithi017 ceph-mon[103001]: pgmap v1167: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 32997/115071 objects degraded (28.675%); 2285/115071 objects misplaced (1.986%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:38.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:38 smithi017 ceph-mon[107556]: pgmap v1167: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 32997/115071 objects degraded (28.675%); 2285/115071 objects misplaced (1.986%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:38.586 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:38.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:38.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:38 smithi189 ceph-mon[90992]: pgmap v1167: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 32997/115071 objects degraded (28.675%); 2285/115071 objects misplaced (1.986%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:38.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.839 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:38.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:38.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:39.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:39.079 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:39.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:39.092 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:39.092 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:39.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:39.111 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:39.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:39.128 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:39.129 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:39.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:39.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:39.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:39.321 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:39.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:39.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:39.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:39.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:39.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32997/115071 objects degraded (28.675%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1134494755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32997/115071 objects degraded (28.675%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1134494755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:39.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:39.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32997/115071 objects degraded (28.675%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1134494755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:39.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:39.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:39.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:40.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:40.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:40.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:40.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:40.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:40 smithi017 ceph-mon[103001]: pgmap v1168: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 32997/114000 objects degraded (28.945%); 2285/114000 objects misplaced (2.004%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:40.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:40 smithi017 ceph-mon[107556]: pgmap v1168: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 32997/114000 objects degraded (28.945%); 2285/114000 objects misplaced (2.004%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:40.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:40.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:40 smithi189 ceph-mon[90992]: pgmap v1168: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 32997/114000 objects degraded (28.945%); 2285/114000 objects misplaced (2.004%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:40.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:40.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:40.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:40.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:40.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:40.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:40.944 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.965 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:40.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:41.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:41.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:41.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:41.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:41.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:41.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.448 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:41.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:41.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:41.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:41.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:41.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:41.851 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:41.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:41.980 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:41.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:42.082 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:42.104 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:42.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:42.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:42.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[107556]: pgmap v1169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 32931/112239 objects degraded (29.340%); 2220/112239 objects misplaced (1.978%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:42.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:42.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:42.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[103001]: pgmap v1169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 32931/112239 objects degraded (29.340%); 2220/112239 objects misplaced (1.978%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:42.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:42 smithi189 ceph-mon[90992]: pgmap v1169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 32931/112239 objects degraded (29.340%); 2220/112239 objects misplaced (1.978%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T14:47:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:42.757 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:42.757 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 32931/112239 objects degraded (29.340%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1660,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":363,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":37413,"data_bytes":2454686390,"bytes_used":12439732224,"bytes_avail":755520552960,"bytes_total":767960285184,"degraded_objects":32931,"degraded_total":112239,"degraded_ratio":0.2934006895998717,"misplaced_objects":2220,"misplaced_total":112239,"misplaced_ratio":0.019779221126346457,"recovering_objects_per_sec":24,"recovering_bytes_per_sec":1582524,"recovering_keys_per_sec":0,"num_objects_recovered":263,"num_bytes_recovered":17235968,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":215},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (8m)\n [==========================..] (remaining: 38s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:47:42.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.882 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:42.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:43.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:43.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:43.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:43.372 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:43.391 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2428478838' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3053432440' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2428478838' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3053432440' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.474 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:43.497 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:43.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:43.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:43.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:43.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:43.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2428478838' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:47:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3053432440' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:43.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:43.879 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:43.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.108 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:44.129 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:44.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:44.317 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:47:44.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:47:44.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:44.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[103001]: pgmap v1170: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 32795/111354 objects degraded (29.451%); 2220/111354 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[103001]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32795/111354 objects degraded (29.451%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[107556]: pgmap v1170: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 32795/111354 objects degraded (29.451%); 2220/111354 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:44.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[107556]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:47:44.561 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32795/111354 objects degraded (29.451%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:44.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:44.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:44.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:44 smithi189 ceph-mon[90992]: pgmap v1170: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 32795/111354 objects degraded (29.451%); 2220/111354 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:44 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:44 smithi189 ceph-mon[90992]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:47:44.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32795/111354 objects degraded (29.451%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:44.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:44.797 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:44.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:45.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:45.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:45.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:45.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:45.862 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:45.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:45.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:45.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:46.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:46.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:46.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:46.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:46.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:46 smithi017 ceph-mon[103001]: pgmap v1172: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 32795/109797 objects degraded (29.869%); 2220/109797 objects misplaced (2.022%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:46.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:46 smithi017 ceph-mon[107556]: pgmap v1172: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 32795/109797 objects degraded (29.869%); 2220/109797 objects misplaced (2.022%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.588 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:46.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:46.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:46 smithi189 ceph-mon[90992]: pgmap v1172: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 32795/109797 objects degraded (29.869%); 2220/109797 objects misplaced (2.022%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:47:46.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:46.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:46.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.945 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:46.945 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:46.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:47.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:47.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:47.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:47.214 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:47.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:47:47.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:47.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:47.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:47.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:47.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.627 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:47] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T14:47:47.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:47.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:47.823 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:47.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:48.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:48.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:47:48.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:48.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:48.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:48.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:48.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:48.374 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:48.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:48.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:48.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.664 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:48 smithi189 ceph-mon[90992]: pgmap v1173: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 32590/108441 objects degraded (30.053%); 2154/108441 objects misplaced (1.986%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:48.679 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:48 smithi017 ceph-mon[107556]: pgmap v1173: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 32590/108441 objects degraded (30.053%); 2154/108441 objects misplaced (1.986%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:48.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:48 smithi017 ceph-mon[103001]: pgmap v1173: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 32590/108441 objects degraded (30.053%); 2154/108441 objects misplaced (1.986%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:48.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:48.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:48.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:48.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:48.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:49.345 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:49.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:49.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:49.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:49.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32590/108441 objects degraded (30.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:49.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32590/108441 objects degraded (30.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:49.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32590/108441 objects degraded (30.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:49.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:49.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:49.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:49.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:49.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:49.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:50.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:50.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:50.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:50.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:50.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:50.332 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:47:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:47:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:50.366 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:50.366 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:50.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:50.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:50.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:50.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:50 smithi017 ceph-mon[103001]: pgmap v1174: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 32590/107268 objects degraded (30.382%); 2154/107268 objects misplaced (2.008%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:50.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:50 smithi017 ceph-mon[107556]: pgmap v1174: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 32590/107268 objects degraded (30.382%); 2154/107268 objects misplaced (2.008%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:50.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.695 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:50 smithi189 ceph-mon[90992]: pgmap v1174: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 224 op/s; 32590/107268 objects degraded (30.382%); 2154/107268 objects misplaced (2.008%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:47:50.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:50.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:50.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.965 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:50.965 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:50.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:50.983 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:51.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:51.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:51.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:51.018 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:51.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:51.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:47:51.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:51.038 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:51.055 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:51.056 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:51.072 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:51.073 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:51.090 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:51.090 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:51.133 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.224 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:51.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.381 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:51.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.443 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:47:51.444 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:47:51.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:51.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:51.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.674 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2076111523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2076111523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:51.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2076111523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:51.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:51.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:52.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:52.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:52.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:52.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:52.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:52.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:52.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:52.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:52.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:52.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:52.681 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:52 smithi017 ceph-mon[107556]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32525/105324 objects degraded (30.881%); 2084/105324 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:52 smithi017 ceph-mon[107556]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:52 smithi017 ceph-mon[103001]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32525/105324 objects degraded (30.881%); 2084/105324 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:52 smithi017 ceph-mon[103001]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:52.731 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1175,"stamp":"2024-09-13T14:47:51.264245+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2303625910,"num_objects":35108,"num_object_clones":0,"num_object_copies":105324,"num_objects_missing_on_primary":1135,"num_objects_missing":1135,"num_objects_degraded":32525,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":35108,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":148545,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26860,"num_bytes_recovered":1696303261,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127130,"ondisk_log_size":127130,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11778256,"kb_used_data":6769232,"kb_used_omap":44,"kb_used_meta":5008915,"kb_avail":738182960,"statfs":{"total":767960285184,"available":755899351040,"internally_reserved":0,"allocated":6931693568,"data_stored":6916223879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5129129207},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-151060480,"num_objects":-2305,"num_object_clones":0,"num_object_copies":-6915,"num_objects_missing_on_primary":-130,"num_objects_missing":-130,"num_objects_degraded":-406,"num_objects_misplaced":-136,"num_objects_unfound":0,"num_objects_dirty":-2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2305,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002835"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":613,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427229+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:47:43.427229+0000","last_peered":"2024-09-13T14:47:43.427229+0000","last_clean":"2024-09-13T14:47:43.427229+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:47:43.427229+0000","last_undegraded":"2024-09-13T14:47:43.427229+0000","last_fullsized":"2024-09-13T14:47:43.427229+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":362,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398483+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:47:30.398483+0000","last_peered":"2024-09-13T14:47:30.398483+0000","last_clean":"2024-09-13T14:47:30.398483+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:47:30.398483+0000","last_undegraded":"2024-09-13T14:47:30.398483+0000","last_fullsized":"2024-09-13T14:47:30.398483+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":291,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371703+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:47:30.371703+0000","last_peered":"2024-09-13T14:47:30.371703+0000","last_clean":"2024-09-13T14:47:30.371703+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:47:30.371703+0000","last_undegraded":"2024-09-13T14:47:30.371703+0000","last_fullsized":"2024-09-13T14:47:30.371703+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":598,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388398+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:47:43.388398+0000","last_peered":"2024-09-13T14:47:43.388398+0000","last_clean":"2024-09-13T14:47:43.388398+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:47:43.388398+0000","last_undegraded":"2024-09-13T14:47:43.388398+0000","last_fullsized":"2024-09-13T14:47:43.388398+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":598,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.389329+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:47:43.389329+0000","last_peered":"2024-09-13T14:47:43.389329+0000","last_clean":"2024-09-13T14:47:43.389329+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:47:43.389329+0000","last_undegraded":"2024-09-13T14:47:43.389329+0000","last_fullsized":"2024-09-13T14:47:43.389329+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":331,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398106+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:47:30.398106+0000","last_peered":"2024-09-13T14:47:30.398106+0000","last_clean":"2024-09-13T14:47:30.398106+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:47:30.398106+0000","last_undegraded":"2024-09-13T14:47:30.398106+0000","last_fullsized":"2024-09-13T14:47:30.398106+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":612,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.430198+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:47:43.430198+0000","last_peered":"2024-09-13T14:47:43.430198+0000","last_clean":"2024-09-13T14:47:43.430198+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:47:43.430198+0000","last_undegraded":"2024-09-13T14:47:43.430198+0000","last_fullsized":"2024-09-13T14:47:43.430198+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":353,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376263+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:47:30.376263+0000","last_peered":"2024-09-13T14:47:30.376263+0000","last_clean":"2024-09-13T14:47:30.376263+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:47:30.376263+0000","last_undegraded":"2024-09-13T14:47:30.376263+0000","last_fullsized":"2024-09-13T14:47:30.376263+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"364'9132","reported_seq":19723,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:46.960817+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:47:46.960817+0000","last_peered":"2024-09-13T14:47:46.960817+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:47:46.960817+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9132,"log_dups_size":0,"ondisk_log_size":9132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141426688,"num_objects":2158,"num_object_clones":0,"num_object_copies":6474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3231,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9132,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3487,"num_bytes_recovered":228524032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1174}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":357,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398773+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:47:30.398773+0000","last_peered":"2024-09-13T14:47:30.398773+0000","last_clean":"2024-09-13T14:47:30.398773+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:47:30.398773+0000","last_undegraded":"2024-09-13T14:47:30.398773+0000","last_fullsized":"2024-09-13T14:47:30.398773+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371323+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:47:30.371323+0000","last_peered":"2024-09-13T14:47:30.371323+0000","last_clean":"2024-09-13T14:47:30.371323+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:47:30.371323+0000","last_undegraded":"2024-09-13T14:47:30.371323+0000","last_fullsized":"2024-09-13T14:47:30.371323+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"364'9239","reported_seq":18800,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:47.582123+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:47:47.582123+0000","last_peered":"2024-09-13T14:47:47.582123+0000","last_clean":"2024-09-13T14:47:47.582123+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:47:47.582123+0000","last_undegraded":"2024-09-13T14:47:47.582123+0000","last_fullsized":"2024-09-13T14:47:47.582123+0000","mapping_epoch":326,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7539,"log_dups_size":0,"ondisk_log_size":7539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147390464,"num_objects":2249,"num_object_clones":0,"num_object_copies":6747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9239,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":375,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398463+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:47:30.398463+0000","last_peered":"2024-09-13T14:47:30.398463+0000","last_clean":"2024-09-13T14:47:30.398463+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:47:30.398463+0000","last_undegraded":"2024-09-13T14:47:30.398463+0000","last_fullsized":"2024-09-13T14:47:30.398463+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.379848+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:47:30.379848+0000","last_peered":"2024-09-13T14:47:30.379848+0000","last_clean":"2024-09-13T14:47:30.379848+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:47:30.379848+0000","last_undegraded":"2024-09-13T14:47:30.379848+0000","last_fullsized":"2024-09-13T14:47:30.379848+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"364'9537","reported_seq":9673,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:48.736750+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:47:48.736750+0000","last_peered":"2024-09-13T14:47:48.736750+0000","last_clean":"2024-09-13T14:47:48.736750+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:47:48.736750+0000","last_undegraded":"2024-09-13T14:47:48.736750+0000","last_fullsized":"2024-09-13T14:47:48.736750+0000","mapping_epoch":303,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8637,"log_dups_size":0,"ondisk_log_size":8637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148176896,"num_objects":2261,"num_object_clones":0,"num_object_copies":6783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398408+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:47:30.398408+0000","last_peered":"2024-09-13T14:47:30.398408+0000","last_clean":"2024-09-13T14:47:30.398408+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:47:30.398408+0000","last_undegraded":"2024-09-13T14:47:30.398408+0000","last_fullsized":"2024-09-13T14:47:30.398408+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":344,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376593+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:47:30.376593+0000","last_peered":"2024-09-13T14:47:30.376593+0000","last_clean":"2024-09-13T14:47:30.376593+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:47:30.376593+0000","last_undegraded":"2024-09-13T14:47:30.376593+0000","last_fullsized":"2024-09-13T14:47:30.376593+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"364'9330","reported_seq":9466,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:50.039014+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:47:50.039014+0000","last_peered":"2024-09-13T14:47:50.039014+0000","last_clean":"2024-09-13T14:47:50.039014+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:47:50.039014+0000","last_undegraded":"2024-09-13T14:47:50.039014+0000","last_fullsized":"2024-09-13T14:47:50.039014+0000","mapping_epoch":303,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145227776,"num_objects":2216,"num_object_clones":0,"num_object_copies":6648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9330,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":292,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371594+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:47:30.371594+0000","last_peered":"2024-09-13T14:47:30.371594+0000","last_clean":"2024-09-13T14:47:30.371594+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:47:30.371594+0000","last_undegraded":"2024-09-13T14:47:30.371594+0000","last_fullsized":"2024-09-13T14:47:30.371594+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":624,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388831+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:47:43.388831+0000","last_peered":"2024-09-13T14:47:43.388831+0000","last_clean":"2024-09-13T14:47:43.388831+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:47:43.388831+0000","last_undegraded":"2024-09-13T14:47:43.388831+0000","last_fullsized":"2024-09-13T14:47:43.388831+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"364'9304","reported_seq":9438,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:50.487708+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:47:50.487708+0000","last_peered":"2024-09-13T14:47:50.487708+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:47:50.487708+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141426688,"num_objects":2158,"num_object_clones":0,"num_object_copies":6474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9304,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-3475}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":475,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398011+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:47:30.398011+0000","last_peered":"2024-09-13T14:47:30.398011+0000","last_clean":"2024-09-13T14:47:30.398011+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:47:30.398011+0000","last_undegraded":"2024-09-13T14:47:30.398011+0000","last_fullsized":"2024-09-13T14:47:30.398011+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":622,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427366+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:47:43.427366+0000","last_peered":"2024-09-13T14:47:43.427366+0000","last_clean":"2024-09-13T14:47:43.427366+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:47:43.427366+0000","last_undegraded":"2024-09-13T14:47:43.427366+0000","last_fullsized":"2024-09-13T14:47:43.427366+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380068+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:47:30.380068+0000","last_peered":"2024-09-13T14:47:30.380068+0000","last_clean":"2024-09-13T14:47:30.380068+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:47:30.380068+0000","last_undegraded":"2024-09-13T14:47:30.380068+0000","last_fullsized":"2024-09-13T14:47:30.380068+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"364'8968","reported_seq":9104,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:50.013405+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:47:50.013405+0000","last_peered":"2024-09-13T14:47:50.013405+0000","last_clean":"2024-09-13T14:47:50.013405+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:47:50.013405+0000","last_undegraded":"2024-09-13T14:47:50.013405+0000","last_fullsized":"2024-09-13T14:47:50.013405+0000","mapping_epoch":303,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139329536,"num_objects":2126,"num_object_clones":0,"num_object_copies":6378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8968,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":179,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371408+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:47:30.371408+0000","last_peered":"2024-09-13T14:47:30.371408+0000","last_clean":"2024-09-13T14:47:30.371408+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:47:30.371408+0000","last_undegraded":"2024-09-13T14:47:30.371408+0000","last_fullsized":"2024-09-13T14:47:30.371408+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":365,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376379+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:47:30.376379+0000","last_peered":"2024-09-13T14:47:30.376379+0000","last_clean":"2024-09-13T14:47:30.376379+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:47:30.376379+0000","last_undegraded":"2024-09-13T14:47:30.376379+0000","last_fullsized":"2024-09-13T14:47:30.376379+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":77,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380598+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:47:30.380598+0000","last_peered":"2024-09-13T14:47:30.380598+0000","last_clean":"2024-09-13T14:47:30.380598+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:47:30.380598+0000","last_undegraded":"2024-09-13T14:47:30.380598+0000","last_fullsized":"2024-09-13T14:47:30.380598+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"364'9224","reported_seq":19502,"reported_epoch":364,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:50.392380+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:47:50.392380+0000","last_peered":"2024-09-13T14:47:50.392380+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:47:50.392380+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7324,"log_dups_size":0,"ondisk_log_size":7324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143392768,"num_objects":2188,"num_object_clones":0,"num_object_copies":6564,"num_objects_missing_on_primary":1135,"num_objects_missing":1135,"num_objects_degraded":1510,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":2188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9224,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3519,"num_bytes_recovered":230621184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1135},{"shards":"0,3,4","objects":949},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.379661+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:47:30.379661+0000","last_peered":"2024-09-13T14:47:30.379661+0000","last_clean":"2024-09-13T14:47:30.379661+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:47:30.379661+0000","last_undegraded":"2024-09-13T14:47:30.379661+0000","last_fullsized":"2024-09-13T14:47:30.379661+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380678+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:47:30.380678+0000","last_peered":"2024-09-13T14:47:30.380678+0000","last_clean":"2024-09-13T14:47:30.380678+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:47:30.380678+0000","last_undegraded":"2024-09-13T14:47:30.380678+0000","last_fullsized":"2024-09-13T14:47:30.380678+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"364'8992","reported_seq":9128,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:50.009909+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:47:50.009909+0000","last_peered":"2024-09-13T14:47:50.009909+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:47:50.009909+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7592,"log_dups_size":0,"ondisk_log_size":7592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141950976,"num_objects":2166,"num_object_clones":0,"num_object_copies":6498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8992,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-3300}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":172,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371935+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:47:30.371935+0000","last_peered":"2024-09-13T14:47:30.371935+0000","last_clean":"2024-09-13T14:47:30.371935+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:47:30.371935+0000","last_undegraded":"2024-09-13T14:47:30.371935+0000","last_fullsized":"2024-09-13T14:47:30.371935+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":615,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.431808+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:47:43.431808+0000","last_peered":"2024-09-13T14:47:43.431808+0000","last_clean":"2024-09-13T14:47:43.431808+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:47:43.431808+0000","last_undegraded":"2024-09-13T14:47:43.431808+0000","last_fullsized":"2024-09-13T14:47:43.431808+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":576,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.414125+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:47:43.414125+0000","last_peered":"2024-09-13T14:47:43.414125+0000","last_clean":"2024-09-13T14:47:43.414125+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:47:43.414125+0000","last_undegraded":"2024-09-13T14:47:43.414125+0000","last_fullsized":"2024-09-13T14:47:43.414125+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"364'9058","reported_seq":9134,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:47.952846+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:47:47.952846+0000","last_peered":"2024-09-13T14:47:47.952846+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:47:47.952846+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141688832,"num_objects":2162,"num_object_clones":0,"num_object_copies":6486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9058,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-3334}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":466,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.400596+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:47:30.400596+0000","last_peered":"2024-09-13T14:47:30.400596+0000","last_clean":"2024-09-13T14:47:30.400596+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:47:30.400596+0000","last_undegraded":"2024-09-13T14:47:30.400596+0000","last_fullsized":"2024-09-13T14:47:30.400596+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":888,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371831+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:47:30.371831+0000","last_peered":"2024-09-13T14:47:30.371831+0000","last_clean":"2024-09-13T14:47:30.371831+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:47:30.371831+0000","last_undegraded":"2024-09-13T14:47:30.371831+0000","last_fullsized":"2024-09-13T14:47:30.371831+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":568,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.426999+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:47:43.426999+0000","last_peered":"2024-09-13T14:47:43.426999+0000","last_clean":"2024-09-13T14:47:43.426999+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:47:43.426999+0000","last_undegraded":"2024-09-13T14:47:43.426999+0000","last_fullsized":"2024-09-13T14:47:43.426999+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":330,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376743+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:47:30.376743+0000","last_peered":"2024-09-13T14:47:30.376743+0000","last_clean":"2024-09-13T14:47:30.376743+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:47:30.376743+0000","last_undegraded":"2024-09-13T14:47:30.376743+0000","last_fullsized":"2024-09-13T14:47:30.376743+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"364'9381","reported_seq":9515,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:50.075152+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:47:50.075152+0000","last_peered":"2024-09-13T14:47:50.075152+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:47:50.075152+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142016512,"num_objects":2167,"num_object_clones":0,"num_object_copies":6501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9381,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3513}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":422,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413457+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:47:43.413457+0000","last_peered":"2024-09-13T14:47:43.413457+0000","last_clean":"2024-09-13T14:47:43.413457+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:47:43.413457+0000","last_undegraded":"2024-09-13T14:47:43.413457+0000","last_fullsized":"2024-09-13T14:47:43.413457+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":754,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.400590+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:47:30.400590+0000","last_peered":"2024-09-13T14:47:30.400590+0000","last_clean":"2024-09-13T14:47:30.400590+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:47:30.400590+0000","last_undegraded":"2024-09-13T14:47:30.400590+0000","last_fullsized":"2024-09-13T14:47:30.400590+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":617,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388521+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:47:43.388521+0000","last_peered":"2024-09-13T14:47:43.388521+0000","last_clean":"2024-09-13T14:47:43.388521+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:47:43.388521+0000","last_undegraded":"2024-09-13T14:47:43.388521+0000","last_fullsized":"2024-09-13T14:47:43.388521+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":622,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.389558+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:47:43.389558+0000","last_peered":"2024-09-13T14:47:43.389558+0000","last_clean":"2024-09-13T14:47:43.389558+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:47:43.389558+0000","last_undegraded":"2024-09-13T14:47:43.389558+0000","last_fullsized":"2024-09-13T14:47:43.389558+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"364'9413","reported_seq":19090,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:50.438210+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:47:50.438210+0000","last_peered":"2024-09-13T14:47:50.438210+0000","last_clean":"2024-09-13T14:47:50.438210+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:47:50.438210+0000","last_undegraded":"2024-09-13T14:47:50.438210+0000","last_fullsized":"2024-09-13T14:47:50.438210+0000","mapping_epoch":330,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9413,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":495,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398179+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:47:30.398179+0000","last_peered":"2024-09-13T14:47:30.398179+0000","last_clean":"2024-09-13T14:47:30.398179+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:47:30.398179+0000","last_undegraded":"2024-09-13T14:47:30.398179+0000","last_fullsized":"2024-09-13T14:47:30.398179+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":586,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427108+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:47:43.427108+0000","last_peered":"2024-09-13T14:47:43.427108+0000","last_clean":"2024-09-13T14:47:43.427108+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:47:43.427108+0000","last_undegraded":"2024-09-13T14:47:43.427108+0000","last_fullsized":"2024-09-13T14:47:43.427108+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":554,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413265+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:47:43.413265+0000","last_peered":"2024-09-13T14:47:43.413265+0000","last_clean":"2024-09-13T14:47:43.413265+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:47:43.413265+0000","last_undegraded":"2024-09-13T14:47:43.413265+0000","last_fullsized":"2024-09-13T14:47:43.413265+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371474+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:47:30.371474+0000","last_peered":"2024-09-13T14:47:30.371474+0000","last_clean":"2024-09-13T14:47:30.371474+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:47:30.371474+0000","last_undegraded":"2024-09-13T14:47:30.371474+0000","last_fullsized":"2024-09-13T14:47:30.371474+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"364'9035","reported_seq":9170,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:50.450993+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:47:50.450993+0000","last_peered":"2024-09-13T14:47:50.450993+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:47:50.450993+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'2000","ondisk_log_start":"307'2000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143851520,"num_objects":2195,"num_object_clones":0,"num_object_copies":6585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9035,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-3314}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":172,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371358+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:47:30.371358+0000","last_peered":"2024-09-13T14:47:30.371358+0000","last_clean":"2024-09-13T14:47:30.371358+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:47:30.371358+0000","last_undegraded":"2024-09-13T14:47:30.371358+0000","last_fullsized":"2024-09-13T14:47:30.371358+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":979,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371941+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:47:30.371941+0000","last_peered":"2024-09-13T14:47:30.371941+0000","last_clean":"2024-09-13T14:47:30.371941+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:47:30.371941+0000","last_undegraded":"2024-09-13T14:47:30.371941+0000","last_fullsized":"2024-09-13T14:47:30.371941+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380244+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:47:30.380244+0000","last_peered":"2024-09-13T14:47:30.380244+0000","last_clean":"2024-09-13T14:47:30.380244+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:47:30.380244+0000","last_undegraded":"2024-09-13T14:47:30.380244+0000","last_fullsized":"2024-09-13T14:47:30.380244+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":575,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413356+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:47:43.413356+0000","last_peered":"2024-09-13T14:47:43.413356+0000","last_clean":"2024-09-13T14:47:43.413356+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:47:43.413356+0000","last_undegraded":"2024-09-13T14:47:43.413356+0000","last_fullsized":"2024-09-13T14:47:43.413356+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"364'9159","reported_seq":18575,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:48.646010+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:47:48.646010+0000","last_peered":"2024-09-13T14:47:48.646010+0000","last_clean":"2024-09-13T14:47:48.646010+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:47:48.646010+0000","last_undegraded":"2024-09-13T14:47:48.646010+0000","last_fullsized":"2024-09-13T14:47:48.646010+0000","mapping_epoch":337,"log_start":"306'500","ondisk_log_start":"306'500","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8659,"log_dups_size":0,"ondisk_log_size":8659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144244736,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9159,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":363,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376722+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:47:30.376722+0000","last_peered":"2024-09-13T14:47:30.376722+0000","last_clean":"2024-09-13T14:47:30.376722+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:47:30.376722+0000","last_undegraded":"2024-09-13T14:47:30.376722+0000","last_fullsized":"2024-09-13T14:47:30.376722+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":309,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371612+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:47:30.371612+0000","last_peered":"2024-09-13T14:47:30.371612+0000","last_clean":"2024-09-13T14:47:30.371612+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:47:30.371612+0000","last_undegraded":"2024-09-13T14:47:30.371612+0000","last_fullsized":"2024-09-13T14:47:30.371612+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1154,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413563+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:47:43.413563+0000","last_peered":"2024-09-13T14:47:43.413563+0000","last_clean":"2024-09-13T14:47:43.413563+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:47:43.413563+0000","last_undegraded":"2024-09-13T14:47:43.413563+0000","last_fullsized":"2024-09-13T14:47:43.413563+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":556,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.420300+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:47:43.420300+0000","last_peered":"2024-09-13T14:47:43.420300+0000","last_clean":"2024-09-13T14:47:43.420300+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:47:43.420300+0000","last_undegraded":"2024-09-13T14:47:43.420300+0000","last_fullsized":"2024-09-13T14:47:43.420300+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"364'9071","reported_seq":9207,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:50.510254+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:47:50.510254+0000","last_peered":"2024-09-13T14:47:50.510254+0000","last_clean":"2024-09-13T14:47:50.510254+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:47:50.510254+0000","last_undegraded":"2024-09-13T14:47:50.510254+0000","last_fullsized":"2024-09-13T14:47:50.510254+0000","mapping_epoch":303,"log_start":"307'2000","ondisk_log_start":"307'2000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139788288,"num_objects":2133,"num_object_clones":0,"num_object_copies":6399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9071,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":363,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376389+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:47:30.376389+0000","last_peered":"2024-09-13T14:47:30.376389+0000","last_clean":"2024-09-13T14:47:30.376389+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:47:30.376389+0000","last_undegraded":"2024-09-13T14:47:30.376389+0000","last_fullsized":"2024-09-13T14:47:30.376389+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":565,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413437+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:47:43.413437+0000","last_peered":"2024-09-13T14:47:43.413437+0000","last_clean":"2024-09-13T14:47:43.413437+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:47:43.413437+0000","last_undegraded":"2024-09-13T14:47:43.413437+0000","last_fullsized":"2024-09-13T14:47:43.413437+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":758,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398486+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:47:30.398486+0000","last_peered":"2024-09-13T14:47:30.398486+0000","last_clean":"2024-09-13T14:47:30.398486+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:47:30.398486+0000","last_undegraded":"2024-09-13T14:47:30.398486+0000","last_fullsized":"2024-09-13T14:47:30.398486+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":600,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388809+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:47:43.388809+0000","last_peered":"2024-09-13T14:47:43.388809+0000","last_clean":"2024-09-13T14:47:43.388809+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:47:43.388809+0000","last_undegraded":"2024-09-13T14:47:43.388809+0000","last_fullsized":"2024-09-13T14:47:43.388809+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"364'9431","reported_seq":9567,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:48.813724+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:47:48.813724+0000","last_peered":"2024-09-13T14:47:48.813724+0000","last_clean":"2024-09-13T14:47:48.813724+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:47:48.813724+0000","last_undegraded":"2024-09-13T14:47:48.813724+0000","last_fullsized":"2024-09-13T14:47:48.813724+0000","mapping_epoch":303,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8631,"log_dups_size":0,"ondisk_log_size":8631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9431,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380429+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:47:30.380429+0000","last_peered":"2024-09-13T14:47:30.380429+0000","last_clean":"2024-09-13T14:47:30.380429+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:47:30.380429+0000","last_undegraded":"2024-09-13T14:47:30.380429+0000","last_fullsized":"2024-09-13T14:47:30.380429+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":601,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.430428+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:47:43.430428+0000","last_peered":"2024-09-13T14:47:43.430428+0000","last_clean":"2024-09-13T14:47:43.430428+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:47:43.430428+0000","last_undegraded":"2024-09-13T14:47:43.430428+0000","last_fullsized":"2024-09-13T14:47:43.430428+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":941,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413926+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:47:43.413926+0000","last_peered":"2024-09-13T14:47:43.413926+0000","last_clean":"2024-09-13T14:47:43.413926+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:47:43.413926+0000","last_undegraded":"2024-09-13T14:47:43.413926+0000","last_fullsized":"2024-09-13T14:47:43.413926+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":598,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.431684+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:47:43.431684+0000","last_peered":"2024-09-13T14:47:43.431684+0000","last_clean":"2024-09-13T14:47:43.431684+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:47:43.431684+0000","last_undegraded":"2024-09-13T14:47:43.431684+0000","last_fullsized":"2024-09-13T14:47:43.431684+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"364'9324","reported_seq":20699,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:47.726138+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:47:47.726138+0000","last_peered":"2024-09-13T14:47:47.726138+0000","last_clean":"2024-09-13T14:47:47.726138+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:47:47.726138+0000","last_undegraded":"2024-09-13T14:47:47.726138+0000","last_fullsized":"2024-09-13T14:47:47.726138+0000","mapping_epoch":332,"log_start":"306'200","ondisk_log_start":"306'200","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143458328,"num_objects":2190,"num_object_clones":0,"num_object_copies":6570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2190,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9324,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":452,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427165+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:47:43.427165+0000","last_peered":"2024-09-13T14:47:43.427165+0000","last_clean":"2024-09-13T14:47:43.427165+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:47:43.427165+0000","last_undegraded":"2024-09-13T14:47:43.427165+0000","last_fullsized":"2024-09-13T14:47:43.427165+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":636,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.389219+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:47:43.389219+0000","last_peered":"2024-09-13T14:47:43.389219+0000","last_clean":"2024-09-13T14:47:43.389219+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:47:43.389219+0000","last_undegraded":"2024-09-13T14:47:43.389219+0000","last_fullsized":"2024-09-13T14:47:43.389219+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":782,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376851+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:47:30.376851+0000","last_peered":"2024-09-13T14:47:30.376851+0000","last_clean":"2024-09-13T14:47:30.376851+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:47:30.376851+0000","last_undegraded":"2024-09-13T14:47:30.376851+0000","last_fullsized":"2024-09-13T14:47:30.376851+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.379774+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:47:30.379774+0000","last_peered":"2024-09-13T14:47:30.379774+0000","last_clean":"2024-09-13T14:47:30.379774+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:47:30.379774+0000","last_undegraded":"2024-09-13T14:47:30.379774+0000","last_fullsized":"2024-09-13T14:47:30.379774+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380269+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:47:30.380269+0000","last_peered":"2024-09-13T14:47:30.380269+0000","last_clean":"2024-09-13T14:47:30.380269+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:47:30.380269+0000","last_undegraded":"2024-09-13T14:47:30.380269+0000","last_fullsized":"2024-09-13T14:47:30.380269+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":73,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.379945+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:47:30.379945+0000","last_peered":"2024-09-13T14:47:30.379945+0000","last_clean":"2024-09-13T14:47:30.379945+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:47:30.379945+0000","last_undegraded":"2024-09-13T14:47:30.379945+0000","last_fullsized":"2024-09-13T14:47:30.379945+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":552,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.415586+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:47:43.415586+0000","last_peered":"2024-09-13T14:47:43.415586+0000","last_clean":"2024-09-13T14:47:43.415586+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:47:43.415586+0000","last_undegraded":"2024-09-13T14:47:43.415586+0000","last_fullsized":"2024-09-13T14:47:43.415586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":363,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.398283+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:47:30.398283+0000","last_peered":"2024-09-13T14:47:30.398283+0000","last_clean":"2024-09-13T14:47:30.398283+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:47:30.398283+0000","last_undegraded":"2024-09-13T14:47:30.398283+0000","last_fullsized":"2024-09-13T14:47:30.398283+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371468+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:47:30.371468+0000","last_peered":"2024-09-13T14:47:30.371468+0000","last_clean":"2024-09-13T14:47:30.371468+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:47:30.371468+0000","last_undegraded":"2024-09-13T14:47:30.371468+0000","last_fullsized":"2024-09-13T14:47:30.371468+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":557,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.426885+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:47:43.426885+0000","last_peered":"2024-09-13T14:47:43.426885+0000","last_clean":"2024-09-13T14:47:43.426885+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:47:43.426885+0000","last_undegraded":"2024-09-13T14:47:43.426885+0000","last_fullsized":"2024-09-13T14:47:43.426885+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":608,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.389561+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:47:43.389561+0000","last_peered":"2024-09-13T14:47:43.389561+0000","last_clean":"2024-09-13T14:47:43.389561+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:47:43.389561+0000","last_undegraded":"2024-09-13T14:47:43.389561+0000","last_fullsized":"2024-09-13T14:47:43.389561+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":613,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.431633+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:47:43.431633+0000","last_peered":"2024-09-13T14:47:43.431633+0000","last_clean":"2024-09-13T14:47:43.431633+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:47:43.431633+0000","last_undegraded":"2024-09-13T14:47:43.431633+0000","last_fullsized":"2024-09-13T14:47:43.431633+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":361,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.376783+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:47:30.376783+0000","last_peered":"2024-09-13T14:47:30.376783+0000","last_clean":"2024-09-13T14:47:30.376783+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:47:30.376783+0000","last_undegraded":"2024-09-13T14:47:30.376783+0000","last_fullsized":"2024-09-13T14:47:30.376783+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":630,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.431731+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:47:43.431731+0000","last_peered":"2024-09-13T14:47:43.431731+0000","last_clean":"2024-09-13T14:47:43.431731+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:47:43.431731+0000","last_undegraded":"2024-09-13T14:47:43.431731+0000","last_fullsized":"2024-09-13T14:47:43.431731+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":601,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.389450+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:47:43.389450+0000","last_peered":"2024-09-13T14:47:43.389450+0000","last_clean":"2024-09-13T14:47:43.389450+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:47:43.389450+0000","last_undegraded":"2024-09-13T14:47:43.389450+0000","last_fullsized":"2024-09-13T14:47:43.389450+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":577,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413283+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:47:43.413283+0000","last_peered":"2024-09-13T14:47:43.413283+0000","last_clean":"2024-09-13T14:47:43.413283+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:47:43.413283+0000","last_undegraded":"2024-09-13T14:47:43.413283+0000","last_fullsized":"2024-09-13T14:47:43.413283+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":626,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427275+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:47:43.427275+0000","last_peered":"2024-09-13T14:47:43.427275+0000","last_clean":"2024-09-13T14:47:43.427275+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:47:43.427275+0000","last_undegraded":"2024-09-13T14:47:43.427275+0000","last_fullsized":"2024-09-13T14:47:43.427275+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.387446+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:47:30.387446+0000","last_peered":"2024-09-13T14:47:30.387446+0000","last_clean":"2024-09-13T14:47:30.387446+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:47:30.387446+0000","last_undegraded":"2024-09-13T14:47:30.387446+0000","last_fullsized":"2024-09-13T14:47:30.387446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":523,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413675+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:47:43.413675+0000","last_peered":"2024-09-13T14:47:43.413675+0000","last_clean":"2024-09-13T14:47:43.413675+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:47:43.413675+0000","last_undegraded":"2024-09-13T14:47:43.413675+0000","last_fullsized":"2024-09-13T14:47:43.413675+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":604,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388728+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:47:43.388728+0000","last_peered":"2024-09-13T14:47:43.388728+0000","last_clean":"2024-09-13T14:47:43.388728+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:47:43.388728+0000","last_undegraded":"2024-09-13T14:47:43.388728+0000","last_fullsized":"2024-09-13T14:47:43.388728+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.380206+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:47:30.380206+0000","last_peered":"2024-09-13T14:47:30.380206+0000","last_clean":"2024-09-13T14:47:30.380206+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:47:30.380206+0000","last_undegraded":"2024-09-13T14:47:30.380206+0000","last_fullsized":"2024-09-13T14:47:30.380206+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371651+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:47:30.371651+0000","last_peered":"2024-09-13T14:47:30.371651+0000","last_clean":"2024-09-13T14:47:30.371651+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:47:30.371651+0000","last_undegraded":"2024-09-13T14:47:30.371651+0000","last_fullsized":"2024-09-13T14:47:30.371651+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.387360+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:47:30.387360+0000","last_peered":"2024-09-13T14:47:30.387360+0000","last_clean":"2024-09-13T14:47:30.387360+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:47:30.387360+0000","last_undegraded":"2024-09-13T14:47:30.387360+0000","last_fullsized":"2024-09-13T14:47:30.387360+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":295,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.371756+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:47:30.371756+0000","last_peered":"2024-09-13T14:47:30.371756+0000","last_clean":"2024-09-13T14:47:30.371756+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:47:30.371756+0000","last_undegraded":"2024-09-13T14:47:30.371756+0000","last_fullsized":"2024-09-13T14:47:30.371756+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":622,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427077+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:47:43.427077+0000","last_peered":"2024-09-13T14:47:43.427077+0000","last_clean":"2024-09-13T14:47:43.427077+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:47:43.427077+0000","last_undegraded":"2024-09-13T14:47:43.427077+0000","last_fullsized":"2024-09-13T14:47:43.427077+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":609,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427548+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:47:43.427548+0000","last_peered":"2024-09-13T14:47:43.427548+0000","last_clean":"2024-09-13T14:47:43.427548+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:47:43.427548+0000","last_undegraded":"2024-09-13T14:47:43.427548+0000","last_fullsized":"2024-09-13T14:47:43.427548+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.379646+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:47:30.379646+0000","last_peered":"2024-09-13T14:47:30.379646+0000","last_clean":"2024-09-13T14:47:30.379646+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:47:30.379646+0000","last_undegraded":"2024-09-13T14:47:30.379646+0000","last_fullsized":"2024-09-13T14:47:30.379646+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":614,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388523+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:47:43.388523+0000","last_peered":"2024-09-13T14:47:43.388523+0000","last_clean":"2024-09-13T14:47:43.388523+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:47:43.388523+0000","last_undegraded":"2024-09-13T14:47:43.388523+0000","last_fullsized":"2024-09-13T14:47:43.388523+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2298937368,"num_objects":35080,"num_object_clones":0,"num_object_copies":105240,"num_objects_missing_on_primary":1135,"num_objects_missing":1135,"num_objects_degraded":32525,"num_objects_misplaced":2084,"num_objects_unfound":0,"num_objects_dirty":35080,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147598,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26803,"num_bytes_recovered":1696137216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6857502720,"data_stored":6857490504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124998,"ondisk_log_size":124998,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397940,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622140,"kb_used_data":730292,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":92123012,"statfs":{"total":95995035648,"available":94333964288,"internally_reserved":0,"allocated":747819008,"data_stored":745869741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594168,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1663336,"kb_used_data":1073440,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":92081816,"statfs":{"total":95995035648,"available":94291779584,"internally_reserved":0,"allocated":1099202560,"data_stored":1097273047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665622,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520360,"kb_used_data":191968,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93224792,"statfs":{"total":95995035648,"available":95462187008,"internally_reserved":0,"allocated":196575232,"data_stored":194642050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986624,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455028,"kb_used_data":563500,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":92290124,"statfs":{"total":95995035648,"available":94505086976,"internally_reserved":0,"allocated":577024000,"data_stored":575082831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697184,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965540,"kb_used_data":1211548,"kb_used_omap":4,"kb_used_meta":753979,"kb_avail":91779612,"statfs":{"total":95995035648,"available":93982322688,"internally_reserved":0,"allocated":1240625152,"data_stored":1238696123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":772074571},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860710,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768172,"kb_used_data":1151076,"kb_used_omap":8,"kb_used_meta":617079,"kb_avail":91976980,"statfs":{"total":95995035648,"available":94184427520,"internally_reserved":0,"allocated":1178701824,"data_stored":1176765141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":631889740},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020382,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1305248,"kb_used_data":991960,"kb_used_omap":9,"kb_used_meta":313270,"kb_avail":92439904,"statfs":{"total":95995035648,"available":94658461696,"internally_reserved":0,"allocated":1015767040,"data_stored":1013841484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":320789317},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187772,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1478432,"kb_used_data":855448,"kb_used_omap":7,"kb_used_meta":622968,"kb_avail":92266720,"statfs":{"total":95995035648,"available":94481121280,"internally_reserved":0,"allocated":875978752,"data_stored":874053462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":637919563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":866910208,"data_stored":866910208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":999817216,"data_stored":999817216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1173688320,"data_stored":1173684248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1230700544,"data_stored":1230700544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":568000512,"data_stored":568000512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":191369216,"data_stored":191365144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1092091904,"data_stored":1092091904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":734924800,"data_stored":734920728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:47:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:52 smithi189 ceph-mon[90992]: pgmap v1175: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32525/105324 objects degraded (30.881%); 2084/105324 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:52 smithi189 ceph-mon[90992]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:52.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:52.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:53.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:53.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:53.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:53.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:53.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:53.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.576 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:47:53.576 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:47:53.576 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:47:53.576 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:47:53.576 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:47:53.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:53.698 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:53.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:54.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:54.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:54.291 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:47:54.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:54 smithi017 ceph-mon[103001]: pgmap v1176: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 32389/104514 objects degraded (30.990%); 2084/104514 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:54.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32389/104514 objects degraded (30.990%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:54.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:54 smithi017 ceph-mon[107556]: pgmap v1176: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 32389/104514 objects degraded (30.990%); 2084/104514 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:54.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32389/104514 objects degraded (30.990%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:54.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.567 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:54.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.602 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:54.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:54.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:54.733 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:54 smithi189 ceph-mon[90992]: pgmap v1176: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 227 op/s; 32389/104514 objects degraded (30.990%); 2084/104514 objects misplaced (1.994%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:54.733 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32389/104514 objects degraded (30.990%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:54.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:54.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:55.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:55.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:55.201 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:55.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.229 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:55.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:55.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:55.416 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:47:55.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:55.486 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:55.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:55.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:55.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:55.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:47:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:55.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:55.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:55.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:55.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:56.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.115 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:56.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:56.363 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:56.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.434 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97448 8.2 8:432ac46e:::benchmark_data_smithi017_253885_object6107:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 372.417538928, 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7480373149999999, 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97448 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.435 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013456+0000", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013454+0000", 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013463+0000", 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2749999999999995e-06 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013478+0000", 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4827999999999999e-05 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.013484+0000", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0120000000000002e-06 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.732932+0000", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71944847000000001 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.740403+0000", 2024-09-13T14:47:56.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074705850000000001 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.786063+0000", 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.045660457000000002 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.794357+0000", 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0082942269999999995 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.642588+0000", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8482307659999999 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.649051+0000", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064624849999999996 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.662391+0000", 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01334071 2024-09-13T14:47:56.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.705020+0000", 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.042628591 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.736739+0000", 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031718728000000002 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.738882+0000", 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021435120000000002 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.742772+0000", 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038903879999999998 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.761493+0000", 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01872095 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.442 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97452 8.2 8:439122c9:::benchmark_data_smithi017_253885_object6111:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 372.401680107, 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9971937799999999, 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97452 2024-09-13T14:47:56.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029315+0000", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029313+0000", 2024-09-13T14:47:56.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029321+0000", 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3140000000000004e-06 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029335+0000", 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3889e-05 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.029342+0000", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0430000000000002e-06 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.741208+0000", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.711865953 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.748863+0000", 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076552809999999999 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.862098+0000", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.11323446500000001 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.870656+0000", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085582220000000007 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.905504+0000", 2024-09-13T14:47:56.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0348476689999999 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.912694+0000", 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071900239999999997 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.926156+0000", 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013462386 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.999476+0000", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073319630999999996 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.001708+0000", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022318780000000001 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.004718+0000", 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030106780000000001 2024-09-13T14:47:56.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.008705+0000", 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039868530000000003 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.026509+0000", 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017803531000000001 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97472 8.2 8:4bd78c32:::benchmark_data_smithi017_253885_object6131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 372.25975580599999, 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8783020779999999, 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97472 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171239+0000", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171238+0000", 2024-09-13T14:47:56.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171245+0000", 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6980000000000001e-06 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171256+0000", 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1698e-05 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.171266+0000", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0113e-05 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.871813+0000", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70054635300000001 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.879204+0000", 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073909620000000001 2024-09-13T14:47:56.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.920565+0000", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.041361235000000003 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.929027+0000", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084622120000000002 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.969886+0000", 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0408586409999998 2024-09-13T14:47:56.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.977252+0000", 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073662780000000004 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:46.989408+0000", 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012155454 2024-09-13T14:47:56.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.002450+0000", 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01304277 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.028897+0000", 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.026447114000000001 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.031314+0000", 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024162010000000002 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.035036+0000", 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037223960000000002 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.049541+0000", 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014505267000000001 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.458 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:97476 8.2 8:489530ca:::benchmark_data_smithi017_253885_object6135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 372.20721755599999, 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0112176900000001, 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.459 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 97476 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223777+0000", 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223776+0000", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223785+0000", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223799+0000", 2024-09-13T14:47:56.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4086e-05 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.223807+0000", 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1669999999999999e-06 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.883412+0000", 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.65960507599999996 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.890980+0000", 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075679470000000002 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.959455+0000", 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068474876000000004 2024-09-13T14:47:56.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.464 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.464 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.464 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:44.968466+0000", 2024-09-13T14:47:56.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0090107199999999998 2024-09-13T14:47:56.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.105941+0000", 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.137475292 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.113301+0000", 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073604889999999996 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.125657+0000", 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012355319999999999 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.162775+0000", 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.037118233 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.176717+0000", 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013941808 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.216950+0000", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040233172999999997 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.220608+0000", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036583179999999998 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:47.234995+0000", 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01438681 2024-09-13T14:47:56.467 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98595 8.2 8:4695e799:::benchmark_data_smithi017_253885_object7254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 360.190577082, 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.745307597, 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.468 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98595 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240418+0000", 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240416+0000", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240425+0000", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0589999999999992e-06 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240440+0000", 2024-09-13T14:47:56.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4465e-05 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.240447+0000", 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7150000000000001e-06 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.413119+0000", 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.172672935 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.471 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.420808+0000", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076884919999999999 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.489162+0000", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068353763999999997 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.497299+0000", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081373120000000007 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.875937+0000", 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3786378049999999 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.884179+0000", 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0082421079999999997 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.901112+0000", 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016932966000000001 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.954696+0000", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.053583700999999997 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.959435+0000", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047390230000000002 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.961626+0000", 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00219162 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.965435+0000", 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038089930000000001 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.985726+0000", 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020290269 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98605 8.2 8:41a3b6c0:::benchmark_data_smithi017_253885_object7264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.475 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 359.97688462500003, 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7582236789999999, 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98605 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454110+0000", 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454109+0000", 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454115+0000", 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9090000000000003e-06 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454128+0000", 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2670999999999999e-05 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.454135+0000", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.776e-06 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.498066+0000", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.043930780000000003 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.507224+0000", 2024-09-13T14:47:56.479 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091585580000000007 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.553871+0000", 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046647010000000003 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.480 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.563511+0000", 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0096394859999999992 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.090575+0000", 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5270640310000001 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.097314+0000", 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006738979 2024-09-13T14:47:56.481 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.109230+0000", 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011916456000000001 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.184619+0000", 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075389251000000004 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.482 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.187198+0000", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025785669999999999 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.191458+0000", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004260107 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.195445+0000", 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039864619999999996 2024-09-13T14:47:56.483 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.212334+0000", 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016889441000000002 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98654 8.2 8:4498e4e3:::benchmark_data_smithi017_253885_object7313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:47:56.484 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 359.44358358699998, 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7594836640000002, 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98654 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.485 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987411+0000", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987410+0000", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987416+0000", 2024-09-13T14:47:56.486 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9050000000000002e-06 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987427+0000", 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1325000000000001e-05 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:56.987432+0000", 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7650000000000001e-06 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.487 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.068663+0000", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.081231138999999994 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.076300+0000", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076368790000000001 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.143490+0000", 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.067189789 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.488 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.152408+0000", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.008918367 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.615689+0000", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4632809 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.623737+0000", 2024-09-13T14:47:56.489 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080478649999999995 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.639371+0000", 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015633463 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.722883+0000", 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083512102000000005 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.490 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.725216+0000", 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002332804 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.728339+0000", 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031238519999999999 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.732036+0000", 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003697035 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.491 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.746895+0000", 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014858675999999999 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98659 8.2 8:4652443c:::benchmark_data_smithi017_253885_object7318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 359.16126769599998, 2024-09-13T14:47:56.492 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8136869770000001, 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98659 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.493 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269727+0000", 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269726+0000", 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.494 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269732+0000", 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.959e-06 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269744+0000", 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1746000000000001e-05 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.495 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.269750+0000", 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8560000000000003e-06 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.292615+0000", 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022865963999999999 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.300310+0000", 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076944789999999997 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.496 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.496894+0000", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19658410300000001 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:57.505348+0000", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084539760000000002 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.964988+0000", 2024-09-13T14:47:56.497 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.459640442 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.971768+0000", 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0067792420000000004 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.983698+0000", 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011930690000000001 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.498 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.058423+0000", 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.074724156 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.060669+0000", 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022467469999999999 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.063945+0000", 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0032759719999999998 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.499 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.067424+0000", 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003479102 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.083414+0000", 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015989894000000001 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.500 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98859 8.2 8:440b55f0:::benchmark_data_smithi017_253885_object7518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 357.864359143, 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8518238469999999, 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98859 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.501 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566636+0000", 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566634+0000", 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.502 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566642+0000", 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8890000000000002e-06 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566654+0000", 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1341e-05 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.566659+0000", 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8570000000000003e-06 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.732707+0000", 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.16604880599999999 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.741658+0000", 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0089508469999999996 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.504 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.812527+0000", 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.070868908999999994 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:58.827305+0000", 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014777521 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.505 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.320283+0000", 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4929778950000001 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.327304+0000", 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070219059999999996 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.339435+0000", 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012130448 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.506 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.383514+0000", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.044078719000000002 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.387513+0000", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039998100000000003 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.391087+0000", 2024-09-13T14:47:56.507 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035737669999999998 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.397274+0000", 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061866619999999999 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.418460+0000", 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021185874 2024-09-13T14:47:56.508 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98909 8.2 8:4a9c011b:::benchmark_data_smithi017_253885_object7568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 357.401270754, 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0264528579999999, 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.509 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98909 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029724+0000", 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.510 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029723+0000", 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029730+0000", 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0090000000000001e-06 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.511 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029743+0000", 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3865e-05 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.029751+0000", 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4429999999999997e-06 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.216354+0000", 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18660349200000001 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.512 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.226081+0000", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0097264250000000003 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.295483+0000", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.069402162000000003 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.309279+0000", 2024-09-13T14:47:56.513 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013796294000000001 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.921663+0000", 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6123836869999999 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.928931+0000", 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072678009999999999 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.514 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.946660+0000", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017729041000000001 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.982389+0000", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035728754000000001 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.988132+0000", 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057438530000000002 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.515 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.026695+0000", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.038562415000000003 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.031498+0000", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004802906 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.056177+0000", 2024-09-13T14:47:56.516 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024679390999999998 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:98993 8.2 8:46d18c53:::benchmark_data_smithi017_253885_object7652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 356.66102266899998, 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8958036410000001, 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.517 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 98993 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769972+0000", 2024-09-13T14:47:56.518 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769970+0000", 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769978+0000", 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.525e-06 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769990+0000", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2349e-05 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.769995+0000", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1240000000000004e-06 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.864919+0000", 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094923341999999994 2024-09-13T14:47:56.520 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.873053+0000", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081339970000000004 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.920149+0000", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047096617 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:59.930004+0000", 2024-09-13T14:47:56.521 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0098543599999999995 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.599878+0000", 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6698743540000001 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.607075+0000", 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071964400000000001 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.522 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.619997+0000", 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012922703000000001 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.638895+0000", 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018897450999999999 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.639635+0000", 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00074067800000000004 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.523 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.642118+0000", 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024828799999999998 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.645888+0000", 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037692960000000001 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.524 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.665776+0000", 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019888385000000001 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99143 8.2 8:4f3b130b:::benchmark_data_smithi017_253885_object7802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 355.79388900999999, 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7602121840000002, 2024-09-13T14:47:56.525 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99143 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.526 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637106+0000", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637104+0000", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637113+0000", 2024-09-13T14:47:56.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4479999999999994e-06 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637128+0000", 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5591e-05 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.637136+0000", 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9789999999999993e-06 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.842593+0000", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20545687000000001 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.852984+0000", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010390893 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.003182+0000", 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.150197414 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.012316+0000", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.009134171 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.221151+0000", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2088349360000001 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.227828+0000", 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066769289999999999 2024-09-13T14:47:56.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.239426+0000", 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011598308 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.311405+0000", 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.071979498000000003 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.341966+0000", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030560744000000001 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.373730+0000", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031763668000000002 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.377836+0000", 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041062160000000002 2024-09-13T14:47:56.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.397318+0000", 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019482152999999999 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99146 8.2 8:499fc9a0:::benchmark_data_smithi017_253885_object7805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 355.774101745, 2024-09-13T14:47:56.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7721449680000001, 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99146 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.534 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656893+0000", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656892+0000", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656899+0000", 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1189999999999999e-06 2024-09-13T14:47:56.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656912+0000", 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2971e-05 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.656918+0000", 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.013329+0000", 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.35641155200000002 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.022916+0000", 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095863830000000004 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.075239+0000", 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052323242999999998 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.539 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.091427+0000", 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016188428000000001 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.279679+0000", 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1882518329999998 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.287334+0000", 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0076550480000000002 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.299647+0000", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012312963999999999 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.398525+0000", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.098877815999999993 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.401134+0000", 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026091700000000001 2024-09-13T14:47:56.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.404614+0000", 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034802000000000001 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.408343+0000", 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037289850000000002 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.429038+0000", 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020694791000000001 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99172 8.2 8:4a96f2ce:::benchmark_data_smithi017_253885_object7831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 355.49814829899998, 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8392140829999999, 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.543 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99172 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932847+0000", 2024-09-13T14:47:56.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932844+0000", 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932854+0000", 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2089999999999994e-06 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932868+0000", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4715000000000001e-05 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:00.932878+0000", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1900000000000001e-06 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.137866+0000", 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20498866900000001 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.145900+0000", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080335430000000006 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.197956+0000", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052055782000000002 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.206529+0000", 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085734160000000004 2024-09-13T14:47:56.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.624724+0000", 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.418195297 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.632205+0000", 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074804520000000003 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.548 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.646855+0000", 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014649970999999999 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.744121+0000", 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.097266562000000001 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.746620+0000", 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002498648 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.748823+0000", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002202673 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.752588+0000", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003765876 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.772061+0000", 2024-09-13T14:47:56.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019472322 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99178 8.2 8:45566d88:::benchmark_data_smithi017_253885_object7837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 355.035490901, 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.922370785, 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.551 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99178 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395504+0000", 2024-09-13T14:47:56.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395502+0000", 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395510+0000", 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9359999999999999e-06 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395525+0000", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4917e-05 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.395532+0000", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9909999999999996e-06 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.426417+0000", 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030884630999999999 2024-09-13T14:47:56.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.434362+0000", 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0079452840000000004 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.628865+0000", 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19450308499999999 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.555 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:01.639250+0000", 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010384581 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.139981+0000", 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5007314940000001 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.147758+0000", 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077768819999999997 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.556 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.160527+0000", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012769019 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.235879+0000", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075351561999999997 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.295287+0000", 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.059408395000000003 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.557 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.299011+0000", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037238200000000001 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.303047+0000", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040359360000000004 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:04.317875+0000", 2024-09-13T14:47:56.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014828182000000001 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99338 8.2 8:41ebf48b:::benchmark_data_smithi017_253885_object7997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 353.49308013199999, 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8585652349999999, 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.559 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99338 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937915+0000", 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937913+0000", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937921+0000", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9990000000000008e-06 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937936+0000", 2024-09-13T14:47:56.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4851999999999999e-05 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.937942+0000", 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2129999999999996e-06 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.971043+0000", 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.033100282000000002 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:02.979114+0000", 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080716690000000001 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.126251+0000", 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14713672799999999 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:03.134698+0000", 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084468640000000001 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.563 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.611327+0000", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4766293250000002 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.622937+0000", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011609647000000001 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.635618+0000", 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012681143000000001 2024-09-13T14:47:56.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.708910+0000", 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073292369999999996 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.709651+0000", 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00074075200000000001 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.775615+0000", 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.065963663000000006 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.779356+0000", 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037407 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.796480+0000", 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017124550999999998 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.566 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:99635 8.2 8:4844b8d9:::benchmark_data_smithi017_253885_object8294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 351.172571721, 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7339699610000001, 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 99635 2024-09-13T14:47:56.567 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258423+0000", 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258422+0000", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258428+0000", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0569999999999998e-06 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258437+0000", 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1039999999999996e-06 2024-09-13T14:47:56.569 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.258442+0000", 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2359999999999998e-06 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.441876+0000", 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18343456899999999 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.570 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.450359+0000", 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084833749999999996 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.532946+0000", 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082586406000000001 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:05.543896+0000", 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010949898 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.827699+0000", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2838030069999999 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.835173+0000", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074745250000000001 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.847362+0000", 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012188516 2024-09-13T14:47:56.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.926601+0000", 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079239233000000006 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.927360+0000", 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00075908799999999999 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.968013+0000", 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.040653214 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.971873+0000", 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003859974 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:07.992393+0000", 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020519936999999999 2024-09-13T14:47:56.574 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100002 8.2 8:4bf97488:::benchmark_data_smithi017_253885_object8661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 347.20040120700003, 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8636257819999997, 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100002 2024-09-13T14:47:56.575 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230594+0000", 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.576 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230593+0000", 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230599+0000", 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1630000000000001e-06 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230608+0000", 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5680000000000005e-06 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.577 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.230613+0000", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.622e-06 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.288032+0000", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.057418652000000001 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.298634+0000", 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010602504 2024-09-13T14:47:56.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.412808+0000", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.114174079 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.421456+0000", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0086479599999999997 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.995933+0000", 2024-09-13T14:47:56.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5744767720000001 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.002777+0000", 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068444159999999999 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.014676+0000", 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011898283000000001 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.056025+0000", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.041349823000000001 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.059989+0000", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039637860000000004 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.061952+0000", 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019627899999999998 2024-09-13T14:47:56.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.069233+0000", 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072812229999999999 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.094220+0000", 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024986313 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.582 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100010 8.2 8:496ef423:::benchmark_data_smithi017_253885_object8669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 347.12335741599998, 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.744854245, 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100010 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.583 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307638+0000", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307636+0000", 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.584 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307643+0000", 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8909999999999998e-06 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307656+0000", 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2782000000000001e-05 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.585 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.307662+0000", 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9039999999999997e-06 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.454671+0000", 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14700880499999999 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.462987+0000", 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0083164420000000003 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.535549+0000", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.072561821999999998 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.544006+0000", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084573340000000004 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.963417+0000", 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4194113389999998 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.970485+0000", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070679580000000001 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:11.983275+0000", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012789327 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.026413+0000", 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.043137847999999999 2024-09-13T14:47:56.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.029413+0000", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030000489999999999 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.030380+0000", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00096762300000000003 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.034053+0000", 2024-09-13T14:47:56.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036728160000000002 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.052492+0000", 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018438754000000002 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:100020 8.2 8:480f1fee:::benchmark_data_smithi017_253885_object8679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-09-13T14:47:56.590 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 346.90999246400003, 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.817903598, 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 100020 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:47:56.591 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521003+0000", 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521001+0000", 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521009+0000", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3149999999999992e-06 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521023+0000", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4157e-05 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.521031+0000", 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3989999999999998e-06 2024-09-13T14:47:56.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.544782+0000", 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023750996 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for missing object", 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.552261+0000", 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074796719999999997 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.598595+0000", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046333107999999998 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for rw locks", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:09.607874+0000", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0092794469999999997 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.170772+0000", 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.562898235 2024-09-13T14:47:56.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.180804+0000", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010031833 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.200140+0000", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019335871000000001 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.254006+0000", 2024-09-13T14:47:56.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.053865937000000003 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.267136+0000", 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013130372 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.316011+0000", 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048874362999999997 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:47:56.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.320353+0000", 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043422640000000002 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:42:12.338906+0000", 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018553284 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:47:56.598 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:47:56.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:56.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[103001]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 32389/102975 objects degraded (31.453%); 2084/102975 objects misplaced (2.024%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[103001]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:47:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[107556]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 32389/102975 objects degraded (31.453%); 2084/102975 objects misplaced (2.024%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:56 smithi017 ceph-mon[107556]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:47:56.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:56 smithi189 ceph-mon[90992]: pgmap v1177: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 32389/102975 objects degraded (31.453%); 2084/102975 objects misplaced (2.024%); 1.4 MiB/s, 23 objects/s recovering 2024-09-13T14:47:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:56 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/770061606' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:56.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:56 smithi189 ceph-mon[90992]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:47:56.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:56.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:56.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:56.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:56.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:57.086 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.113 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:57.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:57.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:47:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:47:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:47:57] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:47:57.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:57.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:57.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:57.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:57.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:57.756 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:57.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:57.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:57.915 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:47:57.916 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:57.933 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:47:57.933 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:57.951 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:47:57.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:57.969 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:47:57.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:57.978 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:57.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:47:57.990 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:58.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:58.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:58.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:47:58.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:58.031 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:47:58.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:58.051 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:47:58.051 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:58.059 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:58.108 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[103001]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 32171/101682 objects degraded (31.639%); 2018/101682 objects misplaced (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3771772455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[107556]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 32171/101682 objects degraded (31.639%); 2018/101682 objects misplaced (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3771772455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:58.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.235 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:47:58.235 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:58.235 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:58 smithi189 ceph-mon[90992]: pgmap v1179: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 32171/101682 objects degraded (31.639%); 2018/101682 objects misplaced (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:47:58.235 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3771772455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:58.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.610 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:58.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:58.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:58.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:58.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:58.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:58.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.021 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:59.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:59.232 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 32171/101682 objects degraded (31.639%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:59.232 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:47:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:59.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 32171/101682 objects degraded (31.639%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:59.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:47:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:59.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:59.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:59.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:59.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 32171/101682 objects degraded (31.639%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:47:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:47:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:59.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:59.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:47:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:59.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:59.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:47:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:59.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:59.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:47:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:00.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:00.155 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:00.155 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:00.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:00 smithi017 ceph-mon[103001]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 32171/100239 objects degraded (32.094%); 2018/100239 objects misplaced (2.013%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:00.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:00 smithi017 ceph-mon[107556]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 32171/100239 objects degraded (32.094%); 2018/100239 objects misplaced (2.013%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:00.451 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:00.451 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:00 smithi189 ceph-mon[90992]: pgmap v1180: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 32171/100239 objects degraded (32.094%); 2018/100239 objects misplaced (2.013%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:00.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:00.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:00.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:00.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:48:00.888 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:00.889 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:48:00.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:01.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:01.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:01.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:01.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:01.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:01.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.330 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:01.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:01.396 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:01.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:01.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:01.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:01.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:01.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:01.842 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:01.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:02.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:02.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:02.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.465 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:02.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:02.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:02.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:02 smithi017 ceph-mon[103001]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:02.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:02 smithi017 ceph-mon[103001]: pgmap v1181: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32100/98415 objects degraded (32.617%); 1948/98415 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:02.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:02 smithi017 ceph-mon[107556]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:02.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:02 smithi017 ceph-mon[107556]: pgmap v1181: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32100/98415 objects degraded (32.617%); 1948/98415 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:02.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:02.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:02 smithi189 ceph-mon[90992]: from='client.36663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:02.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:02 smithi189 ceph-mon[90992]: pgmap v1181: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 32100/98415 objects degraded (32.617%); 1948/98415 objects misplaced (1.979%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:02.708 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1180,"stamp":"2024-09-13T14:47:59.266393+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2192542390,"num_objects":33413,"num_object_clones":0,"num_object_copies":100239,"num_objects_missing_on_primary":1065,"num_objects_missing":1065,"num_objects_degraded":32171,"num_objects_misplaced":2018,"num_objects_unfound":0,"num_objects_dirty":33413,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":150240,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27068,"num_bytes_recovered":1709934749,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127125,"ondisk_log_size":127125,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11539116,"kb_used_data":6487712,"kb_used_omap":44,"kb_used_meta":5051283,"kb_avail":738422100,"statfs":{"total":767960285184,"available":756144230400,"internally_reserved":0,"allocated":6643417088,"data_stored":6627908164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5172514039},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,3,0,0,0,2,1,4,5,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-153550848,"num_objects":-2343,"num_object_clones":0,"num_object_copies":-7029,"num_objects_missing_on_primary":-135,"num_objects_missing":-135,"num_objects_degraded":-419,"num_objects_misplaced":-136,"num_objects_unfound":0,"num_objects_dirty":-2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2343,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":18219008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002716"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":615,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417569+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:47:55.417569+0000","last_peered":"2024-09-13T14:47:55.417569+0000","last_clean":"2024-09-13T14:47:55.417569+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:47:55.417569+0000","last_undegraded":"2024-09-13T14:47:55.417569+0000","last_fullsized":"2024-09-13T14:47:55.417569+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":363,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418497+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:47:43.418497+0000","last_peered":"2024-09-13T14:47:43.418497+0000","last_clean":"2024-09-13T14:47:43.418497+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:47:43.418497+0000","last_undegraded":"2024-09-13T14:47:43.418497+0000","last_fullsized":"2024-09-13T14:47:43.418497+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":292,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388670+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:47:43.388670+0000","last_peered":"2024-09-13T14:47:43.388670+0000","last_clean":"2024-09-13T14:47:43.388670+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:47:43.388670+0000","last_undegraded":"2024-09-13T14:47:43.388670+0000","last_fullsized":"2024-09-13T14:47:43.388670+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419135+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:47:55.419135+0000","last_peered":"2024-09-13T14:47:55.419135+0000","last_clean":"2024-09-13T14:47:55.419135+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:47:55.419135+0000","last_undegraded":"2024-09-13T14:47:55.419135+0000","last_fullsized":"2024-09-13T14:47:55.419135+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420947+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:47:55.420947+0000","last_peered":"2024-09-13T14:47:55.420947+0000","last_clean":"2024-09-13T14:47:55.420947+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:47:55.420947+0000","last_undegraded":"2024-09-13T14:47:55.420947+0000","last_fullsized":"2024-09-13T14:47:55.420947+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":332,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418342+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:47:43.418342+0000","last_peered":"2024-09-13T14:47:43.418342+0000","last_clean":"2024-09-13T14:47:43.418342+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:47:43.418342+0000","last_undegraded":"2024-09-13T14:47:43.418342+0000","last_fullsized":"2024-09-13T14:47:43.418342+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":614,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.418354+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:47:55.418354+0000","last_peered":"2024-09-13T14:47:55.418354+0000","last_clean":"2024-09-13T14:47:55.418354+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:47:55.418354+0000","last_undegraded":"2024-09-13T14:47:55.418354+0000","last_fullsized":"2024-09-13T14:47:55.418354+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":354,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427047+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:47:43.427047+0000","last_peered":"2024-09-13T14:47:43.427047+0000","last_clean":"2024-09-13T14:47:43.427047+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:47:43.427047+0000","last_undegraded":"2024-09-13T14:47:43.427047+0000","last_fullsized":"2024-09-13T14:47:43.427047+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"365'9273","reported_seq":20292,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:57.083978+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:47:57.083978+0000","last_peered":"2024-09-13T14:47:57.083978+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:47:57.083978+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9273,"log_dups_size":0,"ondisk_log_size":9273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9273,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3629,"num_bytes_recovered":237830144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":1032}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":358,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418712+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:47:43.418712+0000","last_peered":"2024-09-13T14:47:43.418712+0000","last_clean":"2024-09-13T14:47:43.418712+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:47:43.418712+0000","last_undegraded":"2024-09-13T14:47:43.418712+0000","last_fullsized":"2024-09-13T14:47:43.418712+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":277,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388491+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:47:43.388491+0000","last_peered":"2024-09-13T14:47:43.388491+0000","last_clean":"2024-09-13T14:47:43.388491+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:47:43.388491+0000","last_undegraded":"2024-09-13T14:47:43.388491+0000","last_fullsized":"2024-09-13T14:47:43.388491+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"365'9396","reported_seq":18958,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:58.023261+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:47:58.023261+0000","last_peered":"2024-09-13T14:47:58.023261+0000","last_clean":"2024-09-13T14:47:58.023261+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:47:58.023261+0000","last_undegraded":"2024-09-13T14:47:58.023261+0000","last_fullsized":"2024-09-13T14:47:58.023261+0000","mapping_epoch":326,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9396,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":376,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418692+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:47:43.418692+0000","last_peered":"2024-09-13T14:47:43.418692+0000","last_clean":"2024-09-13T14:47:43.418692+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:47:43.418692+0000","last_undegraded":"2024-09-13T14:47:43.418692+0000","last_fullsized":"2024-09-13T14:47:43.418692+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432379+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:47:43.432379+0000","last_peered":"2024-09-13T14:47:43.432379+0000","last_clean":"2024-09-13T14:47:43.432379+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:47:43.432379+0000","last_undegraded":"2024-09-13T14:47:43.432379+0000","last_fullsized":"2024-09-13T14:47:43.432379+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"365'9687","reported_seq":9825,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:59.035046+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:47:59.035046+0000","last_peered":"2024-09-13T14:47:59.035046+0000","last_clean":"2024-09-13T14:47:59.035046+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:47:59.035046+0000","last_undegraded":"2024-09-13T14:47:59.035046+0000","last_fullsized":"2024-09-13T14:47:59.035046+0000","mapping_epoch":303,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8587,"log_dups_size":0,"ondisk_log_size":8587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138346496,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9687,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":330,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418625+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:47:43.418625+0000","last_peered":"2024-09-13T14:47:43.418625+0000","last_clean":"2024-09-13T14:47:43.418625+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:47:43.418625+0000","last_undegraded":"2024-09-13T14:47:43.418625+0000","last_fullsized":"2024-09-13T14:47:43.418625+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":345,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427236+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:47:43.427236+0000","last_peered":"2024-09-13T14:47:43.427236+0000","last_clean":"2024-09-13T14:47:43.427236+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:47:43.427236+0000","last_undegraded":"2024-09-13T14:47:43.427236+0000","last_fullsized":"2024-09-13T14:47:43.427236+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"364'9416","reported_seq":9552,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:54.980335+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:47:54.980335+0000","last_peered":"2024-09-13T14:47:54.980335+0000","last_clean":"2024-09-13T14:47:54.980335+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:47:54.980335+0000","last_undegraded":"2024-09-13T14:47:54.980335+0000","last_fullsized":"2024-09-13T14:47:54.980335+0000","mapping_epoch":303,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7516,"log_dups_size":0,"ondisk_log_size":7516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139591680,"num_objects":2130,"num_object_clones":0,"num_object_copies":6390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9416,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":293,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388831+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:47:43.388831+0000","last_peered":"2024-09-13T14:47:43.388831+0000","last_clean":"2024-09-13T14:47:43.388831+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:47:43.388831+0000","last_undegraded":"2024-09-13T14:47:43.388831+0000","last_fullsized":"2024-09-13T14:47:43.388831+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":626,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420389+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:47:55.420389+0000","last_peered":"2024-09-13T14:47:55.420389+0000","last_clean":"2024-09-13T14:47:55.420389+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:47:55.420389+0000","last_undegraded":"2024-09-13T14:47:55.420389+0000","last_fullsized":"2024-09-13T14:47:55.420389+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"365'9395","reported_seq":9531,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:55.666992+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:47:55.666992+0000","last_peered":"2024-09-13T14:47:55.666992+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:47:55.666992+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135462912,"num_objects":2067,"num_object_clones":0,"num_object_copies":6201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9395,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-3566}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":476,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418250+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:47:43.418250+0000","last_peered":"2024-09-13T14:47:43.418250+0000","last_clean":"2024-09-13T14:47:43.418250+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:47:43.418250+0000","last_undegraded":"2024-09-13T14:47:43.418250+0000","last_fullsized":"2024-09-13T14:47:43.418250+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":624,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417069+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:47:55.417069+0000","last_peered":"2024-09-13T14:47:55.417069+0000","last_clean":"2024-09-13T14:47:55.417069+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:47:55.417069+0000","last_undegraded":"2024-09-13T14:47:55.417069+0000","last_fullsized":"2024-09-13T14:47:55.417069+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432621+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:47:43.432621+0000","last_peered":"2024-09-13T14:47:43.432621+0000","last_clean":"2024-09-13T14:47:43.432621+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:47:43.432621+0000","last_undegraded":"2024-09-13T14:47:43.432621+0000","last_fullsized":"2024-09-13T14:47:43.432621+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"364'9042","reported_seq":9179,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.086489+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:47:55.086489+0000","last_peered":"2024-09-13T14:47:55.086489+0000","last_clean":"2024-09-13T14:47:55.086489+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:47:55.086489+0000","last_undegraded":"2024-09-13T14:47:55.086489+0000","last_fullsized":"2024-09-13T14:47:55.086489+0000","mapping_epoch":303,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7542,"log_dups_size":0,"ondisk_log_size":7542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134479872,"num_objects":2052,"num_object_clones":0,"num_object_copies":6156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9042,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":180,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388593+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:47:43.388593+0000","last_peered":"2024-09-13T14:47:43.388593+0000","last_clean":"2024-09-13T14:47:43.388593+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:47:43.388593+0000","last_undegraded":"2024-09-13T14:47:43.388593+0000","last_fullsized":"2024-09-13T14:47:43.388593+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":366,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427184+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:47:43.427184+0000","last_peered":"2024-09-13T14:47:43.427184+0000","last_clean":"2024-09-13T14:47:43.427184+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:47:43.427184+0000","last_undegraded":"2024-09-13T14:47:43.427184+0000","last_fullsized":"2024-09-13T14:47:43.427184+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.434793+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:47:43.434793+0000","last_peered":"2024-09-13T14:47:43.434793+0000","last_clean":"2024-09-13T14:47:43.434793+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:47:43.434793+0000","last_undegraded":"2024-09-13T14:47:43.434793+0000","last_fullsized":"2024-09-13T14:47:43.434793+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"364'9291","reported_seq":19772,"reported_epoch":364,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:55.396553+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:47:55.396553+0000","last_peered":"2024-09-13T14:47:55.396553+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:47:55.396553+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":1065,"num_objects_missing":1065,"num_objects_degraded":1440,"num_objects_misplaced":2018,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9291,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3585,"num_bytes_recovered":234946560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1065},{"shards":"0,3,4","objects":953},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432213+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:47:43.432213+0000","last_peered":"2024-09-13T14:47:43.432213+0000","last_clean":"2024-09-13T14:47:43.432213+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:47:43.432213+0000","last_undegraded":"2024-09-13T14:47:43.432213+0000","last_fullsized":"2024-09-13T14:47:43.432213+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432712+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:47:43.432712+0000","last_peered":"2024-09-13T14:47:43.432712+0000","last_clean":"2024-09-13T14:47:43.432712+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:47:43.432712+0000","last_undegraded":"2024-09-13T14:47:43.432712+0000","last_fullsized":"2024-09-13T14:47:43.432712+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"364'9066","reported_seq":9203,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:55.054401+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:47:55.054401+0000","last_peered":"2024-09-13T14:47:55.054401+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:47:55.054401+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7566,"log_dups_size":0,"ondisk_log_size":7566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9066,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-3374}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":173,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388606+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:47:43.388606+0000","last_peered":"2024-09-13T14:47:43.388606+0000","last_clean":"2024-09-13T14:47:43.388606+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:47:43.388606+0000","last_undegraded":"2024-09-13T14:47:43.388606+0000","last_fullsized":"2024-09-13T14:47:43.388606+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":617,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424672+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:47:55.424672+0000","last_peered":"2024-09-13T14:47:55.424672+0000","last_clean":"2024-09-13T14:47:55.424672+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:47:55.424672+0000","last_undegraded":"2024-09-13T14:47:55.424672+0000","last_fullsized":"2024-09-13T14:47:55.424672+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":576,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.414125+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:47:43.414125+0000","last_peered":"2024-09-13T14:47:43.414125+0000","last_clean":"2024-09-13T14:47:43.414125+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:47:43.414125+0000","last_undegraded":"2024-09-13T14:47:43.414125+0000","last_fullsized":"2024-09-13T14:47:43.414125+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"365'9203","reported_seq":9280,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:58.040766+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:47:58.040766+0000","last_peered":"2024-09-13T14:47:58.040766+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:47:58.040766+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9203,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-3479}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":467,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418364+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:47:43.418364+0000","last_peered":"2024-09-13T14:47:43.418364+0000","last_clean":"2024-09-13T14:47:43.418364+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:47:43.418364+0000","last_undegraded":"2024-09-13T14:47:43.418364+0000","last_fullsized":"2024-09-13T14:47:43.418364+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":889,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388488+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:47:43.388488+0000","last_peered":"2024-09-13T14:47:43.388488+0000","last_clean":"2024-09-13T14:47:43.388488+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:47:43.388488+0000","last_undegraded":"2024-09-13T14:47:43.388488+0000","last_fullsized":"2024-09-13T14:47:43.388488+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":570,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417190+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:47:55.417190+0000","last_peered":"2024-09-13T14:47:55.417190+0000","last_clean":"2024-09-13T14:47:55.417190+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:47:55.417190+0000","last_undegraded":"2024-09-13T14:47:55.417190+0000","last_fullsized":"2024-09-13T14:47:55.417190+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427772+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:47:43.427772+0000","last_peered":"2024-09-13T14:47:43.427772+0000","last_clean":"2024-09-13T14:47:43.427772+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:47:43.427772+0000","last_undegraded":"2024-09-13T14:47:43.427772+0000","last_fullsized":"2024-09-13T14:47:43.427772+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"364'9457","reported_seq":9591,"reported_epoch":364,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:54.741983+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:47:54.741983+0000","last_peered":"2024-09-13T14:47:54.741983+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:47:54.741983+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9457,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3589}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":422,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413457+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:47:43.413457+0000","last_peered":"2024-09-13T14:47:43.413457+0000","last_clean":"2024-09-13T14:47:43.413457+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:47:43.413457+0000","last_undegraded":"2024-09-13T14:47:43.413457+0000","last_fullsized":"2024-09-13T14:47:43.413457+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":755,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418280+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:47:43.418280+0000","last_peered":"2024-09-13T14:47:43.418280+0000","last_clean":"2024-09-13T14:47:43.418280+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:47:43.418280+0000","last_undegraded":"2024-09-13T14:47:43.418280+0000","last_fullsized":"2024-09-13T14:47:43.418280+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":619,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419980+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:47:55.419980+0000","last_peered":"2024-09-13T14:47:55.419980+0000","last_clean":"2024-09-13T14:47:55.419980+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:47:55.419980+0000","last_undegraded":"2024-09-13T14:47:55.419980+0000","last_fullsized":"2024-09-13T14:47:55.419980+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":624,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421093+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:47:55.421093+0000","last_peered":"2024-09-13T14:47:55.421093+0000","last_clean":"2024-09-13T14:47:55.421093+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:47:55.421093+0000","last_undegraded":"2024-09-13T14:47:55.421093+0000","last_fullsized":"2024-09-13T14:47:55.421093+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"365'9477","reported_seq":19156,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.588041+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:47:55.588041+0000","last_peered":"2024-09-13T14:47:55.588041+0000","last_clean":"2024-09-13T14:47:55.588041+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:47:55.588041+0000","last_undegraded":"2024-09-13T14:47:55.588041+0000","last_fullsized":"2024-09-13T14:47:55.588041+0000","mapping_epoch":330,"log_start":"307'2500","ondisk_log_start":"307'2500","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9477,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":496,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418296+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:47:43.418296+0000","last_peered":"2024-09-13T14:47:43.418296+0000","last_clean":"2024-09-13T14:47:43.418296+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:47:43.418296+0000","last_undegraded":"2024-09-13T14:47:43.418296+0000","last_fullsized":"2024-09-13T14:47:43.418296+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":588,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416359+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:47:55.416359+0000","last_peered":"2024-09-13T14:47:55.416359+0000","last_clean":"2024-09-13T14:47:55.416359+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:47:55.416359+0000","last_undegraded":"2024-09-13T14:47:55.416359+0000","last_fullsized":"2024-09-13T14:47:55.416359+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":554,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413265+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:47:43.413265+0000","last_peered":"2024-09-13T14:47:43.413265+0000","last_clean":"2024-09-13T14:47:43.413265+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:47:43.413265+0000","last_undegraded":"2024-09-13T14:47:43.413265+0000","last_fullsized":"2024-09-13T14:47:43.413265+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388783+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:47:43.388783+0000","last_peered":"2024-09-13T14:47:43.388783+0000","last_clean":"2024-09-13T14:47:43.388783+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:47:43.388783+0000","last_undegraded":"2024-09-13T14:47:43.388783+0000","last_fullsized":"2024-09-13T14:47:43.388783+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"365'9107","reported_seq":9244,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:47:55.559737+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:47:55.559737+0000","last_peered":"2024-09-13T14:47:55.559737+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:47:55.559737+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139132928,"num_objects":2123,"num_object_clones":0,"num_object_copies":6369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9107,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-3386}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":173,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388512+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:47:43.388512+0000","last_peered":"2024-09-13T14:47:43.388512+0000","last_clean":"2024-09-13T14:47:43.388512+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:47:43.388512+0000","last_undegraded":"2024-09-13T14:47:43.388512+0000","last_fullsized":"2024-09-13T14:47:43.388512+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":980,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388688+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:47:43.388688+0000","last_peered":"2024-09-13T14:47:43.388688+0000","last_clean":"2024-09-13T14:47:43.388688+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:47:43.388688+0000","last_undegraded":"2024-09-13T14:47:43.388688+0000","last_fullsized":"2024-09-13T14:47:43.388688+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432754+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:47:43.432754+0000","last_peered":"2024-09-13T14:47:43.432754+0000","last_clean":"2024-09-13T14:47:43.432754+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:47:43.432754+0000","last_undegraded":"2024-09-13T14:47:43.432754+0000","last_fullsized":"2024-09-13T14:47:43.432754+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":575,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413356+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:47:43.413356+0000","last_peered":"2024-09-13T14:47:43.413356+0000","last_clean":"2024-09-13T14:47:43.413356+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:47:43.413356+0000","last_undegraded":"2024-09-13T14:47:43.413356+0000","last_fullsized":"2024-09-13T14:47:43.413356+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"365'9300","reported_seq":18718,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:58.989508+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:47:58.989508+0000","last_peered":"2024-09-13T14:47:58.989508+0000","last_clean":"2024-09-13T14:47:58.989508+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:47:58.989508+0000","last_undegraded":"2024-09-13T14:47:58.989508+0000","last_fullsized":"2024-09-13T14:47:58.989508+0000","mapping_epoch":337,"log_start":"306'700","ondisk_log_start":"306'700","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9300,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":364,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.429214+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:47:43.429214+0000","last_peered":"2024-09-13T14:47:43.429214+0000","last_clean":"2024-09-13T14:47:43.429214+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:47:43.429214+0000","last_undegraded":"2024-09-13T14:47:43.429214+0000","last_fullsized":"2024-09-13T14:47:43.429214+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":310,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388578+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:47:43.388578+0000","last_peered":"2024-09-13T14:47:43.388578+0000","last_clean":"2024-09-13T14:47:43.388578+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:47:43.388578+0000","last_undegraded":"2024-09-13T14:47:43.388578+0000","last_fullsized":"2024-09-13T14:47:43.388578+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1154,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413563+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:47:43.413563+0000","last_peered":"2024-09-13T14:47:43.413563+0000","last_clean":"2024-09-13T14:47:43.413563+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:47:43.413563+0000","last_undegraded":"2024-09-13T14:47:43.413563+0000","last_fullsized":"2024-09-13T14:47:43.413563+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":556,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.420300+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:47:43.420300+0000","last_peered":"2024-09-13T14:47:43.420300+0000","last_clean":"2024-09-13T14:47:43.420300+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:47:43.420300+0000","last_undegraded":"2024-09-13T14:47:43.420300+0000","last_fullsized":"2024-09-13T14:47:43.420300+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"365'9135","reported_seq":9273,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.637102+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:47:55.637102+0000","last_peered":"2024-09-13T14:47:55.637102+0000","last_clean":"2024-09-13T14:47:55.637102+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:47:55.637102+0000","last_undegraded":"2024-09-13T14:47:55.637102+0000","last_fullsized":"2024-09-13T14:47:55.637102+0000","mapping_epoch":303,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135593984,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9135,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":364,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.427378+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:47:43.427378+0000","last_peered":"2024-09-13T14:47:43.427378+0000","last_clean":"2024-09-13T14:47:43.427378+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:47:43.427378+0000","last_undegraded":"2024-09-13T14:47:43.427378+0000","last_fullsized":"2024-09-13T14:47:43.427378+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":565,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413437+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:47:43.413437+0000","last_peered":"2024-09-13T14:47:43.413437+0000","last_clean":"2024-09-13T14:47:43.413437+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:47:43.413437+0000","last_undegraded":"2024-09-13T14:47:43.413437+0000","last_fullsized":"2024-09-13T14:47:43.413437+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":759,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418685+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:47:43.418685+0000","last_peered":"2024-09-13T14:47:43.418685+0000","last_clean":"2024-09-13T14:47:43.418685+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:47:43.418685+0000","last_undegraded":"2024-09-13T14:47:43.418685+0000","last_fullsized":"2024-09-13T14:47:43.418685+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":602,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420315+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:47:55.420315+0000","last_peered":"2024-09-13T14:47:55.420315+0000","last_clean":"2024-09-13T14:47:55.420315+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:47:55.420315+0000","last_undegraded":"2024-09-13T14:47:55.420315+0000","last_fullsized":"2024-09-13T14:47:55.420315+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"365'9571","reported_seq":9709,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:58.920704+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:47:58.920704+0000","last_peered":"2024-09-13T14:47:58.920704+0000","last_clean":"2024-09-13T14:47:58.920704+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:47:58.920704+0000","last_undegraded":"2024-09-13T14:47:58.920704+0000","last_fullsized":"2024-09-13T14:47:58.920704+0000","mapping_epoch":303,"log_start":"306'900","ondisk_log_start":"306'900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8671,"log_dups_size":0,"ondisk_log_size":8671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138870784,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9571,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.434717+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:47:43.434717+0000","last_peered":"2024-09-13T14:47:43.434717+0000","last_clean":"2024-09-13T14:47:43.434717+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:47:43.434717+0000","last_undegraded":"2024-09-13T14:47:43.434717+0000","last_fullsized":"2024-09-13T14:47:43.434717+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":603,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421488+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:47:55.421488+0000","last_peered":"2024-09-13T14:47:55.421488+0000","last_clean":"2024-09-13T14:47:55.421488+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:47:55.421488+0000","last_undegraded":"2024-09-13T14:47:55.421488+0000","last_fullsized":"2024-09-13T14:47:55.421488+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":941,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413926+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:47:43.413926+0000","last_peered":"2024-09-13T14:47:43.413926+0000","last_clean":"2024-09-13T14:47:43.413926+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:47:43.413926+0000","last_undegraded":"2024-09-13T14:47:43.413926+0000","last_fullsized":"2024-09-13T14:47:43.413926+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424508+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:47:55.424508+0000","last_peered":"2024-09-13T14:47:55.424508+0000","last_clean":"2024-09-13T14:47:55.424508+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:47:55.424508+0000","last_undegraded":"2024-09-13T14:47:55.424508+0000","last_fullsized":"2024-09-13T14:47:55.424508+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"365'9477","reported_seq":20853,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:57.699420+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:47:57.699420+0000","last_peered":"2024-09-13T14:47:57.699420+0000","last_clean":"2024-09-13T14:47:57.699420+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:47:57.699420+0000","last_undegraded":"2024-09-13T14:47:57.699420+0000","last_fullsized":"2024-09-13T14:47:57.699420+0000","mapping_epoch":332,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9177,"log_dups_size":0,"ondisk_log_size":9177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133431320,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9477,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":454,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417429+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:47:55.417429+0000","last_peered":"2024-09-13T14:47:55.417429+0000","last_clean":"2024-09-13T14:47:55.417429+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:47:55.417429+0000","last_undegraded":"2024-09-13T14:47:55.417429+0000","last_fullsized":"2024-09-13T14:47:55.417429+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":638,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420841+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:47:55.420841+0000","last_peered":"2024-09-13T14:47:55.420841+0000","last_clean":"2024-09-13T14:47:55.420841+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:47:55.420841+0000","last_undegraded":"2024-09-13T14:47:55.420841+0000","last_fullsized":"2024-09-13T14:47:55.420841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":783,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.429287+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:47:43.429287+0000","last_peered":"2024-09-13T14:47:43.429287+0000","last_clean":"2024-09-13T14:47:43.429287+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:47:43.429287+0000","last_undegraded":"2024-09-13T14:47:43.429287+0000","last_fullsized":"2024-09-13T14:47:43.429287+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432306+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:47:43.432306+0000","last_peered":"2024-09-13T14:47:43.432306+0000","last_clean":"2024-09-13T14:47:43.432306+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:47:43.432306+0000","last_undegraded":"2024-09-13T14:47:43.432306+0000","last_fullsized":"2024-09-13T14:47:43.432306+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432828+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:47:43.432828+0000","last_peered":"2024-09-13T14:47:43.432828+0000","last_clean":"2024-09-13T14:47:43.432828+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:47:43.432828+0000","last_undegraded":"2024-09-13T14:47:43.432828+0000","last_fullsized":"2024-09-13T14:47:43.432828+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":74,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432487+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:47:43.432487+0000","last_peered":"2024-09-13T14:47:43.432487+0000","last_clean":"2024-09-13T14:47:43.432487+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:47:43.432487+0000","last_undegraded":"2024-09-13T14:47:43.432487+0000","last_fullsized":"2024-09-13T14:47:43.432487+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":552,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.415586+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:47:43.415586+0000","last_peered":"2024-09-13T14:47:43.415586+0000","last_clean":"2024-09-13T14:47:43.415586+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:47:43.415586+0000","last_undegraded":"2024-09-13T14:47:43.415586+0000","last_fullsized":"2024-09-13T14:47:43.415586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":364,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.418591+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:47:43.418591+0000","last_peered":"2024-09-13T14:47:43.418591+0000","last_clean":"2024-09-13T14:47:43.418591+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:47:43.418591+0000","last_undegraded":"2024-09-13T14:47:43.418591+0000","last_fullsized":"2024-09-13T14:47:43.418591+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":330,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388741+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:47:43.388741+0000","last_peered":"2024-09-13T14:47:43.388741+0000","last_clean":"2024-09-13T14:47:43.388741+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:47:43.388741+0000","last_undegraded":"2024-09-13T14:47:43.388741+0000","last_fullsized":"2024-09-13T14:47:43.388741+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":559,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416138+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:47:55.416138+0000","last_peered":"2024-09-13T14:47:55.416138+0000","last_clean":"2024-09-13T14:47:55.416138+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:47:55.416138+0000","last_undegraded":"2024-09-13T14:47:55.416138+0000","last_fullsized":"2024-09-13T14:47:55.416138+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":610,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419578+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:47:55.419578+0000","last_peered":"2024-09-13T14:47:55.419578+0000","last_clean":"2024-09-13T14:47:55.419578+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:47:55.419578+0000","last_undegraded":"2024-09-13T14:47:55.419578+0000","last_fullsized":"2024-09-13T14:47:55.419578+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":615,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424429+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:47:55.424429+0000","last_peered":"2024-09-13T14:47:55.424429+0000","last_clean":"2024-09-13T14:47:55.424429+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:47:55.424429+0000","last_undegraded":"2024-09-13T14:47:55.424429+0000","last_fullsized":"2024-09-13T14:47:55.424429+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":362,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.429257+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:47:43.429257+0000","last_peered":"2024-09-13T14:47:43.429257+0000","last_clean":"2024-09-13T14:47:43.429257+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:47:43.429257+0000","last_undegraded":"2024-09-13T14:47:43.429257+0000","last_fullsized":"2024-09-13T14:47:43.429257+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":632,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424586+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:47:55.424586+0000","last_peered":"2024-09-13T14:47:55.424586+0000","last_clean":"2024-09-13T14:47:55.424586+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:47:55.424586+0000","last_undegraded":"2024-09-13T14:47:55.424586+0000","last_fullsized":"2024-09-13T14:47:55.424586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":603,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421021+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:47:55.421021+0000","last_peered":"2024-09-13T14:47:55.421021+0000","last_clean":"2024-09-13T14:47:55.421021+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:47:55.421021+0000","last_undegraded":"2024-09-13T14:47:55.421021+0000","last_fullsized":"2024-09-13T14:47:55.421021+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":577,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413283+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:47:43.413283+0000","last_peered":"2024-09-13T14:47:43.413283+0000","last_clean":"2024-09-13T14:47:43.413283+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:47:43.413283+0000","last_undegraded":"2024-09-13T14:47:43.413283+0000","last_fullsized":"2024-09-13T14:47:43.413283+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":628,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416975+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:47:55.416975+0000","last_peered":"2024-09-13T14:47:55.416975+0000","last_clean":"2024-09-13T14:47:55.416975+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:47:55.416975+0000","last_undegraded":"2024-09-13T14:47:55.416975+0000","last_fullsized":"2024-09-13T14:47:55.416975+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.387446+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:47:30.387446+0000","last_peered":"2024-09-13T14:47:30.387446+0000","last_clean":"2024-09-13T14:47:30.387446+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:47:30.387446+0000","last_undegraded":"2024-09-13T14:47:30.387446+0000","last_fullsized":"2024-09-13T14:47:30.387446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":523,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.413675+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:47:43.413675+0000","last_peered":"2024-09-13T14:47:43.413675+0000","last_clean":"2024-09-13T14:47:43.413675+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:47:43.413675+0000","last_undegraded":"2024-09-13T14:47:43.413675+0000","last_fullsized":"2024-09-13T14:47:43.413675+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":606,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420220+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:47:55.420220+0000","last_peered":"2024-09-13T14:47:55.420220+0000","last_clean":"2024-09-13T14:47:55.420220+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:47:55.420220+0000","last_undegraded":"2024-09-13T14:47:55.420220+0000","last_fullsized":"2024-09-13T14:47:55.420220+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432678+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:47:43.432678+0000","last_peered":"2024-09-13T14:47:43.432678+0000","last_clean":"2024-09-13T14:47:43.432678+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:47:43.432678+0000","last_undegraded":"2024-09-13T14:47:43.432678+0000","last_fullsized":"2024-09-13T14:47:43.432678+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":330,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388837+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:47:43.388837+0000","last_peered":"2024-09-13T14:47:43.388837+0000","last_clean":"2024-09-13T14:47:43.388837+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:47:43.388837+0000","last_undegraded":"2024-09-13T14:47:43.388837+0000","last_fullsized":"2024-09-13T14:47:43.388837+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-09-13T14:47:30.387360+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:47:30.387360+0000","last_peered":"2024-09-13T14:47:30.387360+0000","last_clean":"2024-09-13T14:47:30.387360+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:47:30.387360+0000","last_undegraded":"2024-09-13T14:47:30.387360+0000","last_fullsized":"2024-09-13T14:47:30.387360+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":296,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.388760+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:47:43.388760+0000","last_peered":"2024-09-13T14:47:43.388760+0000","last_clean":"2024-09-13T14:47:43.388760+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:47:43.388760+0000","last_undegraded":"2024-09-13T14:47:43.388760+0000","last_fullsized":"2024-09-13T14:47:43.388760+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":624,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416433+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:47:55.416433+0000","last_peered":"2024-09-13T14:47:55.416433+0000","last_clean":"2024-09-13T14:47:55.416433+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:47:55.416433+0000","last_undegraded":"2024-09-13T14:47:55.416433+0000","last_fullsized":"2024-09-13T14:47:55.416433+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":611,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417418+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:47:55.417418+0000","last_peered":"2024-09-13T14:47:55.417418+0000","last_clean":"2024-09-13T14:47:55.417418+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:47:55.417418+0000","last_undegraded":"2024-09-13T14:47:55.417418+0000","last_fullsized":"2024-09-13T14:47:55.417418+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-09-13T14:47:43.432189+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:47:43.432189+0000","last_peered":"2024-09-13T14:47:43.432189+0000","last_clean":"2024-09-13T14:47:43.432189+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:47:43.432189+0000","last_undegraded":"2024-09-13T14:47:43.432189+0000","last_fullsized":"2024-09-13T14:47:43.432189+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":616,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419487+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:47:55.419487+0000","last_peered":"2024-09-13T14:47:55.419487+0000","last_clean":"2024-09-13T14:47:55.419487+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:47:55.419487+0000","last_undegraded":"2024-09-13T14:47:55.419487+0000","last_fullsized":"2024-09-13T14:47:55.419487+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2187853848,"num_objects":33385,"num_object_clones":0,"num_object_copies":100155,"num_objects_missing_on_primary":1065,"num_objects_missing":1065,"num_objects_degraded":32171,"num_objects_misplaced":2018,"num_objects_unfound":0,"num_objects_dirty":33385,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149293,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27011,"num_bytes_recovered":1709768704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6559051776,"data_stored":6559039560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124993,"ondisk_log_size":124993,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397942,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578060,"kb_used_data":682052,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":92167092,"statfs":{"total":95995035648,"available":94379102208,"internally_reserved":0,"allocated":698421248,"data_stored":696464134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594170,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621116,"kb_used_data":1027056,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":92124036,"statfs":{"total":95995035648,"available":94335012864,"internally_reserved":0,"allocated":1051705344,"data_stored":1049767984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665624,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":515516,"kb_used_data":182960,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93229636,"statfs":{"total":95995035648,"available":95467147264,"internally_reserved":0,"allocated":187351040,"data_stored":185410011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986625,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1440500,"kb_used_data":548972,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":92304652,"statfs":{"total":95995035648,"available":94519963648,"internally_reserved":0,"allocated":562147328,"data_stored":560206159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,0,2,1,4,5,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697185,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1937828,"kb_used_data":1183836,"kb_used_omap":4,"kb_used_meta":753979,"kb_avail":91807324,"statfs":{"total":95995035648,"available":94010699776,"internally_reserved":0,"allocated":1212248064,"data_stored":1210319035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":772074571},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860712,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1723648,"kb_used_data":1102388,"kb_used_omap":8,"kb_used_meta":621239,"kb_avail":92021504,"statfs":{"total":95995035648,"available":94230020096,"internally_reserved":0,"allocated":1128845312,"data_stored":1126900782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":636149580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020383,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275424,"kb_used_data":957976,"kb_used_omap":9,"kb_used_meta":317430,"kb_avail":92469728,"statfs":{"total":95995035648,"available":94689001472,"internally_reserved":0,"allocated":980967424,"data_stored":979041868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":325049157},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187774,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447024,"kb_used_data":802472,"kb_used_omap":7,"kb_used_meta":644536,"kb_avail":92298128,"statfs":{"total":95995035648,"available":94513283072,"internally_reserved":0,"allocated":821731328,"data_stored":819798191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":660005195},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":810680320,"data_stored":810680320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":965279744,"data_stored":965279744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1117196288,"data_stored":1117192216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1201405952,"data_stored":1201405952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":555220992,"data_stored":555220992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":181211136,"data_stored":181207064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1043267584,"data_stored":1043267584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":684789760,"data_stored":684785688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:48:02.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:48:02.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:02.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:02.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:02.942 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:03.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:03.096 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:03.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:03.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:03.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:03.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:03.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:03.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:03.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:03.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:03.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:03.710 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:03.710 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:03.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:03.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:03.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:03.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:03.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:04.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:04.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:04.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:04.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:04.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:04.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:04.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:04.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:04 smithi017 ceph-mon[103001]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 31988/97626 objects degraded (32.766%); 1948/97626 objects misplaced (1.995%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:04.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 31988/97626 objects degraded (32.766%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:04.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:04 smithi017 ceph-mon[107556]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 31988/97626 objects degraded (32.766%); 1948/97626 objects misplaced (1.995%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:04.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 31988/97626 objects degraded (32.766%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:04.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:04.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:04.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:04 smithi189 ceph-mon[90992]: pgmap v1182: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 31988/97626 objects degraded (32.766%); 1948/97626 objects misplaced (1.995%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 31988/97626 objects degraded (32.766%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:04.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.130 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:05.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:48:05.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:05.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:05.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:05.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:05.849 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:05.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:05.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:05.946 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:05.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:06.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:06.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:06.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:06.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:06.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:06 smithi017 ceph-mon[107556]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 31988/96189 objects degraded (33.255%); 1948/96189 objects misplaced (2.025%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:06.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:06 smithi017 ceph-mon[103001]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 31988/96189 objects degraded (33.255%); 1948/96189 objects misplaced (2.025%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:06.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:06.413 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:06 smithi189 ceph-mon[90992]: pgmap v1183: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 31988/96189 objects degraded (33.255%); 1948/96189 objects misplaced (2.025%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:48:06.503 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:06.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:06.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:06.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:06.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:06.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:07.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:07.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:07.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:07.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:07.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:07.381 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.407 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:07.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:07.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:07.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.648 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4170744448' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:07.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4170744448' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4170744448' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:07.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:07] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:48:07.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:07.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:07.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:07.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:08.011 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:08.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:08.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:08.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:08.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:08.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:08.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:08.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:08.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:08.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:08.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:08.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:08.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:08 smithi189 ceph-mon[90992]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31775/94830 objects degraded (33.507%); 1878/94830 objects misplaced (1.980%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:08.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:08.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:08 smithi017 ceph-mon[103001]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31775/94830 objects degraded (33.507%); 1878/94830 objects misplaced (1.980%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:08.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:08 smithi017 ceph-mon[107556]: pgmap v1184: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31775/94830 objects degraded (33.507%); 1878/94830 objects misplaced (1.980%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:08.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:08.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.819 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:48:08.819 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190271 7.d 7:bc6c9de1:::benchmark_data_smithi017_220393_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 597.206258252, 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33251491900000002, 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.820 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190271 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:48:08.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036802+0000", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036801+0000", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036809+0000", 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8369999999999997e-06 2024-09-13T14:48:08.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036827+0000", 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8233999999999999e-05 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036835+0000", 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036859+0000", 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3932999999999999e-05 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.036943+0000", 2024-09-13T14:48:08.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4275000000000002e-05 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.037020+0000", 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6403999999999996e-05 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.039717+0000", 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026977440000000002 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045560+0000", 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058428489999999998 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369245+0000", 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32368499099999998 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369277+0000", 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1498999999999998e-05 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369317+0000", 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0759000000000001e-05 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190282 7.d 7:bc4d0eee:::benchmark_data_smithi017_220393_object71148:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 597.19725175200006, 2024-09-13T14:48:08.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.32360401599999999, 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190282 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045809+0000", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045808+0000", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045813+0000", 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4319999999999998e-06 2024-09-13T14:48:08.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045831+0000", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7847e-05 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045837+0000", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3990000000000001e-06 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045875+0000", 2024-09-13T14:48:08.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7481000000000002e-05 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.045984+0000", 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108773 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.046540+0000", 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00055675799999999997 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.050112+0000", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035716860000000001 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.055205+0000", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0050931359999999998 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369356+0000", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31415043199999998 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369382+0000", 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6463000000000001e-05 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369413+0000", 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0812999999999998e-05 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:190303 7.d 7:b1faf55c:::benchmark_data_smithi017_220393_object71169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.834 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 597.16089560900002, 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28739342899999998, 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.835 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 190303 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082165+0000", 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082164+0000", 2024-09-13T14:48:08.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082170+0000", 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2580000000000001e-06 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082183+0000", 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3101e-05 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082190+0000", 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9580000000000001e-06 2024-09-13T14:48:08.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082232+0000", 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2157000000000003e-05 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082315+0000", 2024-09-13T14:48:08.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2902e-05 2024-09-13T14:48:08.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.082401+0000", 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5847999999999995e-05 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085041+0000", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026398659999999998 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.085709+0000", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00066763599999999999 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369484+0000", 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.28377533599999999 2024-09-13T14:48:08.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369521+0000", 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7574999999999999e-05 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:11.369559+0000", 2024-09-13T14:48:08.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7100000000000001e-05 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195365 7.7 7:e23819ca:::benchmark_data_smithi017_220393_object76231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 589.92734247199996, 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.15216712900000001, 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195365 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315718+0000", 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315717+0000", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315726+0000", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8640000000000008e-06 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315738+0000", 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1396999999999999e-05 2024-09-13T14:48:08.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315743+0000", 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5969999999999999e-06 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315786+0000", 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2601000000000001e-05 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.315917+0000", 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000131326 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.316048+0000", 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013117099999999999 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.319756+0000", 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003707307 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.428425+0000", 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.108669617 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467786+0000", 2024-09-13T14:48:08.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039360456000000002 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467847+0000", 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0847999999999997e-05 2024-09-13T14:48:08.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.467885+0000", 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8680000000000002e-05 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.855 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:195370 7.d 7:b9f15733:::benchmark_data_smithi017_220393_object76236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 589.92438545699997, 2024-09-13T14:48:08.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14708705, 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 195370 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.857 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318675+0000", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318674+0000", 2024-09-13T14:48:08.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318680+0000", 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0700000000000003e-06 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318691+0000", 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0502e-05 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318695+0000", 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2270000000000004e-06 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318730+0000", 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5132999999999997e-05 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318848+0000", 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000118114 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.318955+0000", 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000106533 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.321687+0000", 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027322470000000001 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.322829+0000", 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001141853 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465673+0000", 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.14284481299999999 2024-09-13T14:48:08.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465722+0000", 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8504000000000002e-05 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:18.465762+0000", 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0374999999999999e-05 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199290 7.7 7:e5b5f326:::benchmark_data_smithi017_220393_object80156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 584.75381317799997, 2024-09-13T14:48:08.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.20093702499999999, 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.865 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199290 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489248+0000", 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489246+0000", 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489252+0000", 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7209999999999999e-06 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489264+0000", 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2028000000000001e-05 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489270+0000", 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6219999999999997e-06 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489300+0000", 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0456000000000001e-05 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489418+0000", 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117551 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.489507+0000", 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9654999999999995e-05 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.492597+0000", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030899809999999999 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.499941+0000", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007343848 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690098+0000", 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190156834 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690145+0000", 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7094e-05 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.690185+0000", 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9473e-05 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:199306 7.d 7:b9985ddb:::benchmark_data_smithi017_220393_object80172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 584.73606905500003, 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.17325623900000001, 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 199306 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.872 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506992+0000", 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506991+0000", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.506996+0000", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0529999999999997e-06 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507007+0000", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1533e-05 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507013+0000", 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8640000000000001e-06 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507057+0000", 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4322e-05 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507173+0000", 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011561 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.507291+0000", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000117555 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.510005+0000", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027142020000000002 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.511252+0000", 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001246788 2024-09-13T14:48:08.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680183+0000", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.16893149399999999 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680220+0000", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7271999999999998e-05 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:23.680248+0000", 2024-09-13T14:48:08.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7537000000000001e-05 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200609 7.d 7:baf945be:::benchmark_data_smithi017_220393_object81475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 582.81546774399999, 2024-09-13T14:48:08.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34117809199999999, 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200609 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:48:08.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427593+0000", 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427592+0000", 2024-09-13T14:48:08.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427598+0000", 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1979999999999998e-06 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427610+0000", 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.185e-05 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427615+0000", 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.772e-06 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427649+0000", 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3343999999999998e-05 2024-09-13T14:48:08.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427739+0000", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0749999999999997e-05 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.427862+0000", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012259799999999999 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434044+0000", 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061822839999999997 2024-09-13T14:48:08.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435398+0000", 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001354035 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768686+0000", 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33328797199999999 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768733+0000", 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7101999999999997e-05 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768771+0000", 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7586000000000002e-05 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200614 7.d 7:bd88b3c8:::benchmark_data_smithi017_220393_object81480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:48:08.885 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 582.81448867899996, 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.34032926800000002, 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200614 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428572+0000", 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428571+0000", 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428576+0000", 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4469999999999999e-06 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428585+0000", 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.428588+0000", 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5880000000000002e-06 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.430963+0000", 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023746639999999999 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431063+0000", 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000100412 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.431173+0000", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000109085 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.434158+0000", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029855020000000001 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.435507+0000", 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001348471 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768816+0000", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.333309466 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768866+0000", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9571999999999999e-05 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768901+0000", 2024-09-13T14:48:08.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5741000000000002e-05 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200631 7.d 7:b540f86c:::benchmark_data_smithi017_220393_object81497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 582.67611620800005, 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.202045216, 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200631 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566945+0000", 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566943+0000", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566949+0000", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4589999999999997e-06 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566960+0000", 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1046999999999999e-05 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.566966+0000", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9630000000000003e-06 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567008+0000", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1878999999999997e-05 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567095+0000", 2024-09-13T14:48:08.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7145000000000007e-05 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.567169+0000", 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3732000000000005e-05 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.570791+0000", 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036220850000000001 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.571627+0000", 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000835928 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768940+0000", 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19731306200000001 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768970+0000", 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0368000000000001e-05 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.768990+0000", 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9626000000000001e-05 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200634 7.d 7:b80f92f6:::benchmark_data_smithi017_220393_object81500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 582.67046609600004, 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19650853099999999, 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.898 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200634 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572595+0000", 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572594+0000", 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572600+0000", 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1920000000000002e-06 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572611+0000", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1569999999999999e-05 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572616+0000", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2679999999999996e-06 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572656+0000", 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0741000000000002e-05 2024-09-13T14:48:08.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572783+0000", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012668299999999999 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.572938+0000", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000155351 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.575602+0000", 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026634129999999999 2024-09-13T14:48:08.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.577024+0000", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014224999999999999 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769023+0000", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19199869899999999 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769068+0000", 2024-09-13T14:48:08.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5064000000000001e-05 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.769103+0000", 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5173999999999999e-05 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.45566.0:200637 7.d 7:b4355551:::benchmark_data_smithi017_220393_object81503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e278)", 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:48:08.904 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 582.661450973, 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190526902, 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.45566", 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/3827235941", 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 200637 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:48:08.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581610+0000", 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581609+0000", 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581614+0000", 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9139999999999996e-06 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581626+0000", 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1681999999999999e-05 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581632+0000", 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7780000000000004e-06 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581664+0000", 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1585999999999999e-05 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581742+0000", 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8591999999999996e-05 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.581828+0000", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5816000000000006e-05 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584440+0000", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026124199999999998 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.584798+0000", 2024-09-13T14:48:08.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035729199999999999 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772042+0000", 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18724463299999999 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772092+0000", 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9638000000000001e-05 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:38:25.772137+0000", 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4747999999999999e-05 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89676 8.9 8:94dade21:::benchmark_data_smithi017_253885_object89675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 444.54677882499999, 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2982149540000001, 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.911 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89676 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696282+0000", 2024-09-13T14:48:08.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696280+0000", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696320+0000", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9366000000000002e-05 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696398+0000", 2024-09-13T14:48:08.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8026000000000001e-05 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.696404+0000", 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3029999999999996e-06 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.700853+0000", 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0044487659999999998 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.700886+0000", 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2885000000000001e-05 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693311+0000", 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99242506200000002 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693320+0000", 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2830000000000007e-06 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968082+0000", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2747621740000001 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968412+0000", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032966499999999998 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.968877+0000", 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00046530200000000001 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.973656+0000", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047786579999999999 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.978450+0000", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047938470000000004 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.990076+0000", 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011626645999999999 2024-09-13T14:48:08.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.990261+0000", 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018436900000000001 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994497+0000", 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042362190000000003 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89677 8.9 8:975431c6:::benchmark_data_smithi017_253885_object89676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 444.54096538599998, 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3349859369999999, 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89677 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702095+0000", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702094+0000", 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702164+0000", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0179999999999996e-05 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702176+0000", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2001000000000001e-05 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702182+0000", 2024-09-13T14:48:08.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6159999999999998e-06 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702225+0000", 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2599000000000003e-05 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.702234+0000", 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6570000000000003e-06 2024-09-13T14:48:08.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693344+0000", 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.991110088 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693370+0000", 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6111000000000001e-05 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.978555+0000", 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.285184833 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.981496+0000", 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029411260000000001 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.981695+0000", 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019939900000000001 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994573+0000", 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012878069000000001 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994586+0000", 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2709e-05 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.032979+0000", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.03839298 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.033061+0000", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1508000000000006e-05 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037081+0000", 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040207089999999999 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.926 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89693 8.9 8:9608d7fe:::benchmark_data_smithi017_253885_object89692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 444.50705086099998, 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3047559849999999, 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.927 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89693 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736010+0000", 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736008+0000", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736066+0000", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7924999999999999e-05 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736083+0000", 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7312e-05 2024-09-13T14:48:08.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736092+0000", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1139999999999995e-06 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736158+0000", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6256999999999999e-05 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.736170+0000", 2024-09-13T14:48:08.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1457000000000001e-05 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693394+0000", 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95722398200000003 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.693401+0000", 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0899999999999999e-06 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986096+0000", 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2926953990000001 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986273+0000", 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000176534 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.986472+0000", 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019907599999999999 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994627+0000", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081552020000000003 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:45.994633+0000", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0209999999999996e-06 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037134+0000", 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.042501078999999997 2024-09-13T14:48:08.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.037191+0000", 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6669000000000002e-05 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:46.040766+0000", 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0035749890000000002 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94271 8.5 8:a820db60:::benchmark_data_smithi017_253885_object2930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 400.92994629499998, 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96635259200000001, 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94271 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313114+0000", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313112+0000", 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:08.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313121+0000", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9709999999999993e-06 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313138+0000", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6986999999999998e-05 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313145+0000", 2024-09-13T14:48:08.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313203+0000", 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7948000000000002e-05 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.313215+0000", 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2162e-05 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270121+0000", 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95690628200000005 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270239+0000", 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011790699999999999 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.270312+0000", 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3297e-05 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273683+0000", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003371069 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273724+0000", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0321000000000002e-05 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279374+0000", 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0056504349999999997 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279426+0000", 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.219e-05 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279467+0000", 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0595e-05 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94273 8.5 8:a6399fa1:::benchmark_data_smithi017_253885_object2932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:48:08.941 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 400.92454834, 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96120069299999999, 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94273 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:48:08.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318512+0000", 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318511+0000", 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318519+0000", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2490000000000001e-06 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318534+0000", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5048999999999999e-05 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318541+0000", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.63e-06 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318588+0000", 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6711000000000001e-05 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.318596+0000", 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8810000000000002e-06 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273751+0000", 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95515427799999997 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273869+0000", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011859 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.273946+0000", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6808000000000001e-05 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279513+0000", 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055664260000000002 2024-09-13T14:48:08.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279564+0000", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1785e-05 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279640+0000", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5766999999999995e-05 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279681+0000", 2024-09-13T14:48:08.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0914000000000002e-05 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279713+0000", 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2007e-05 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94300 8.5 8:ab0bc9a3:::benchmark_data_smithi017_253885_object2959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:48:08.948 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 400.54461696099997, 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.58689132899999996, 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94300 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698444+0000", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698441+0000", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698452+0000", 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0397000000000001e-05 2024-09-13T14:48:08.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698469+0000", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6835000000000001e-05 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698477+0000", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7150000000000004e-06 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698526+0000", 2024-09-13T14:48:08.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8928000000000003e-05 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.698536+0000", 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4839999999999993e-06 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.276872+0000", 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.57833652099999999 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.276995+0000", 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012297799999999999 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.277112+0000", 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116341 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285181+0000", 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080698880000000008 2024-09-13T14:48:08.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285217+0000", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5676000000000002e-05 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285264+0000", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6406000000000003e-05 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285295+0000", 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1965999999999999e-05 2024-09-13T14:48:08.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285335+0000", 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9610999999999997e-05 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94316 8.5 8:a8a2bf9f:::benchmark_data_smithi017_253885_object2975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 400.36919409699999, 2024-09-13T14:48:08.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.41461699099999999, 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94316 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873867+0000", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873865+0000", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873874+0000", 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4609999999999997e-06 2024-09-13T14:48:08.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873892+0000", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8150000000000001e-05 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873899+0000", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3910000000000003e-06 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873956+0000", 2024-09-13T14:48:08.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7812999999999997e-05 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:27.873969+0000", 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2099999999999999e-05 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279747+0000", 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.40577885899999999 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279852+0000", 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000104879 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.279944+0000", 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1917999999999996e-05 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285224+0000", 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0052794679999999998 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285381+0000", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000157396 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288403+0000", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030220099999999999 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288443+0000", 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9412999999999997e-05 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288484+0000", 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1088999999999999e-05 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:94332 8.5 8:aaf8da73:::benchmark_data_smithi017_253885_object2991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 400.14085836700002, 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.18637316300000001, 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:48:08.962 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 94332 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:48:08.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102202+0000", 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102200+0000", 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102210+0000", 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0145e-05 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102228+0000", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8025000000000001e-05 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102234+0000", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.139e-06 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102290+0000", 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6217000000000001e-05 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.102303+0000", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2911000000000001e-05 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282234+0000", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.179931069 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282310+0000", 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5677000000000001e-05 2024-09-13T14:48:08.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.282375+0000", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5114e-05 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.285415+0000", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030395769999999999 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288351+0000", 2024-09-13T14:48:08.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029367970000000001 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288525+0000", 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017392800000000001 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288555+0000", 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9176999999999999e-05 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:48:08.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:41:28.288576+0000", 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0978000000000001e-05 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:48:08.969 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:48:08.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:08.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:09.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:09.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:09.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:09.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:09.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 31775/94830 objects degraded (33.507%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:09.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 31775/94830 objects degraded (33.507%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:09.465 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:09.466 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 31775/93294 objects degraded (34.059%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1686,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":365,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":31098,"data_bytes":2040826550,"bytes_used":11451711488,"bytes_avail":756508573696,"bytes_total":767960285184,"degraded_objects":31775,"degraded_total":93294,"degraded_ratio":0.34058996291294186,"misplaced_objects":1878,"misplaced_total":93294,"misplaced_ratio":0.020129911891439964,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1468656,"recovering_keys_per_sec":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":232},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (9m)\n [==========================..] (remaining: 41s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:48:09.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:09.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 31775/94830 objects degraded (33.507%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:09.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:09.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:09.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:09.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:48:09.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:09.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:09.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:09.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.119 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:10.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:10.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:10.269 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:48:10.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:48:10.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:10.296 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:10.346 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:48:10.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:10.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:10.422 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:10.422 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:10 smithi189 ceph-mon[90992]: pgmap v1185: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31775/93294 objects degraded (34.059%); 1878/93294 objects misplaced (2.013%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:48:10.422 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2165207095' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:10.422 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3975831925' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:10.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:10.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:10.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[103001]: pgmap v1185: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31775/93294 objects degraded (34.059%); 1878/93294 objects misplaced (2.013%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2165207095' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3975831925' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[107556]: pgmap v1185: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31775/93294 objects degraded (34.059%); 1878/93294 objects misplaced (2.013%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2165207095' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3975831925' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:10.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:10.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:10.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:10.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:11.020 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.102 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:11.102 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:11.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:11.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:11.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:11.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:11.419 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:11.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:11.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:11.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[103001]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:48:11.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:11.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:11.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:11 smithi017 ceph-mon[107556]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:48:11.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.721 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:11 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:11.721 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:11.721 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:11 smithi189 ceph-mon[90992]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:48:11.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:11.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:11.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:12.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:12.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:12.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:12.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:12.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:12 smithi189 ceph-mon[90992]: pgmap v1187: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31699/91560 objects degraded (34.621%); 1806/91560 objects misplaced (1.972%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:12.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:12.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:12.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:12.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:12 smithi017 ceph-mon[107556]: pgmap v1187: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31699/91560 objects degraded (34.621%); 1806/91560 objects misplaced (1.972%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:12 smithi017 ceph-mon[103001]: pgmap v1187: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 31699/91560 objects degraded (34.621%); 1806/91560 objects misplaced (1.972%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:48:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:12.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:12.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:13.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:48:13.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:13.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:13.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.221 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:13.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:48:13.312 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:48:13.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:13.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:13.599 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:13.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:13.724 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:13.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:13.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:13 smithi189 ceph-mon[90992]: pgmap v1188: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31524/90657 objects degraded (34.773%); 1772/90657 objects misplaced (1.955%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:13 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 31699/91560 objects degraded (34.621%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[103001]: pgmap v1188: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31524/90657 objects degraded (34.773%); 1772/90657 objects misplaced (1.955%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 31699/91560 objects degraded (34.621%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:14.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:14.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[107556]: pgmap v1188: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 31524/90657 objects degraded (34.773%); 1772/90657 objects misplaced (1.955%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:14.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:13 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 31699/91560 objects degraded (34.621%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:14.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:14.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:48:14.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:14.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:14.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:14.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:14.425 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:14.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:14.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:14.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.780 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:14.781 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:48:14.833 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:48:14.833 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:48:14.834 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:14.834 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:14.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:14.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:48:14.850 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:14.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:14.875 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:48:14.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:14.891 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:48:14.892 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:14.908 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:48:14.908 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:14.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:48:14.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:14.939 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:48:14.939 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:14.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3429453109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:14.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3429453109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:14.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.990 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:15.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:15.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:15.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3429453109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:15.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:15.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:15.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:15.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:15.620 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:15.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:15.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:15.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:15.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:15 smithi017 ceph-mon[107556]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:15 smithi017 ceph-mon[107556]: pgmap v1189: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 31524/88947 objects degraded (35.441%); 1772/88947 objects misplaced (1.992%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:15 smithi017 ceph-mon[103001]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:15 smithi017 ceph-mon[103001]: pgmap v1189: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 31524/88947 objects degraded (35.441%); 1772/88947 objects misplaced (1.992%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:15.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:15.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:16.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:16.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:16.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:15 smithi189 ceph-mon[90992]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:16.158 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:15 smithi189 ceph-mon[90992]: pgmap v1189: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 31524/88947 objects degraded (35.441%); 1772/88947 objects misplaced (1.992%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:16.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:16.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:16.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:16.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:16.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:16.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:16.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:16.790 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:16.791 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:16.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.039 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:17.059 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1188,"stamp":"2024-09-13T14:48:13.270789+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1983220406,"num_objects":30219,"num_object_clones":0,"num_object_copies":90657,"num_objects_missing_on_primary":820,"num_objects_missing":820,"num_objects_degraded":31524,"num_objects_misplaced":1772,"num_objects_unfound":0,"num_objects_dirty":30219,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":153434,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27515,"num_bytes_recovered":1739229341,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127219,"ondisk_log_size":127219,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11010540,"kb_used_data":5963296,"kb_used_omap":44,"kb_used_meta":5047123,"kb_avail":738950676,"statfs":{"total":767960285184,"available":756685492224,"internally_reserved":0,"allocated":6106415104,"data_stored":6090843404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5168254199},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,2,0,1,1,1,3,4,6,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-152240128,"num_objects":-2323,"num_object_clones":0,"num_object_copies":-6969,"num_objects_missing_on_primary":-174,"num_objects_missing":-174,"num_objects_degraded":-464,"num_objects_misplaced":-176,"num_objects_unfound":0,"num_objects_dirty":-2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2323,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":321,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003144"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":617,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.364813+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:48:10.364813+0000","last_peered":"2024-09-13T14:48:10.364813+0000","last_clean":"2024-09-13T14:48:10.364813+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:48:10.364813+0000","last_undegraded":"2024-09-13T14:48:10.364813+0000","last_fullsized":"2024-09-13T14:48:10.364813+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":364,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417343+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:47:55.417343+0000","last_peered":"2024-09-13T14:47:55.417343+0000","last_clean":"2024-09-13T14:47:55.417343+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:47:55.417343+0000","last_undegraded":"2024-09-13T14:47:55.417343+0000","last_fullsized":"2024-09-13T14:47:55.417343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":293,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414675+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:47:55.414675+0000","last_peered":"2024-09-13T14:47:55.414675+0000","last_clean":"2024-09-13T14:47:55.414675+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:47:55.414675+0000","last_undegraded":"2024-09-13T14:47:55.414675+0000","last_fullsized":"2024-09-13T14:47:55.414675+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419135+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:47:55.419135+0000","last_peered":"2024-09-13T14:47:55.419135+0000","last_clean":"2024-09-13T14:47:55.419135+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:47:55.419135+0000","last_undegraded":"2024-09-13T14:47:55.419135+0000","last_fullsized":"2024-09-13T14:47:55.419135+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420947+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:47:55.420947+0000","last_peered":"2024-09-13T14:47:55.420947+0000","last_clean":"2024-09-13T14:47:55.420947+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:47:55.420947+0000","last_undegraded":"2024-09-13T14:47:55.420947+0000","last_fullsized":"2024-09-13T14:47:55.420947+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":333,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417038+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:47:55.417038+0000","last_peered":"2024-09-13T14:47:55.417038+0000","last_clean":"2024-09-13T14:47:55.417038+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:47:55.417038+0000","last_undegraded":"2024-09-13T14:47:55.417038+0000","last_fullsized":"2024-09-13T14:47:55.417038+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":617,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361200+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:48:10.361200+0000","last_peered":"2024-09-13T14:48:10.361200+0000","last_clean":"2024-09-13T14:48:10.361200+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:48:10.361200+0000","last_undegraded":"2024-09-13T14:48:10.361200+0000","last_fullsized":"2024-09-13T14:48:10.361200+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":355,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.415180+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:47:55.415180+0000","last_peered":"2024-09-13T14:47:55.415180+0000","last_clean":"2024-09-13T14:47:55.415180+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:47:55.415180+0000","last_undegraded":"2024-09-13T14:47:55.415180+0000","last_fullsized":"2024-09-13T14:47:55.415180+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"366'9475","reported_seq":21099,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:12.001272+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:48:12.001272+0000","last_peered":"2024-09-13T14:48:12.001272+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:48:12.001272+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9475,"log_dups_size":0,"ondisk_log_size":9475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118947840,"num_objects":1815,"num_object_clones":0,"num_object_copies":5445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9475,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3830,"num_bytes_recovered":251002880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":831}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":359,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417583+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:47:55.417583+0000","last_peered":"2024-09-13T14:47:55.417583+0000","last_clean":"2024-09-13T14:47:55.417583+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:47:55.417583+0000","last_undegraded":"2024-09-13T14:47:55.417583+0000","last_fullsized":"2024-09-13T14:47:55.417583+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414261+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:47:55.414261+0000","last_peered":"2024-09-13T14:47:55.414261+0000","last_clean":"2024-09-13T14:47:55.414261+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:47:55.414261+0000","last_undegraded":"2024-09-13T14:47:55.414261+0000","last_fullsized":"2024-09-13T14:47:55.414261+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"366'9614","reported_seq":19177,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:12.602726+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:48:12.602726+0000","last_peered":"2024-09-13T14:48:12.602726+0000","last_clean":"2024-09-13T14:48:12.602726+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:48:12.602726+0000","last_undegraded":"2024-09-13T14:48:12.602726+0000","last_fullsized":"2024-09-13T14:48:12.602726+0000","mapping_epoch":326,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122814464,"num_objects":1874,"num_object_clones":0,"num_object_copies":5622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9614,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":377,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417547+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:47:55.417547+0000","last_peered":"2024-09-13T14:47:55.417547+0000","last_clean":"2024-09-13T14:47:55.417547+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:47:55.417547+0000","last_undegraded":"2024-09-13T14:47:55.417547+0000","last_fullsized":"2024-09-13T14:47:55.417547+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422764+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:47:55.422764+0000","last_peered":"2024-09-13T14:47:55.422764+0000","last_clean":"2024-09-13T14:47:55.422764+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:47:55.422764+0000","last_undegraded":"2024-09-13T14:47:55.422764+0000","last_fullsized":"2024-09-13T14:47:55.422764+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"365'9843","reported_seq":9981,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:48:08.860450+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:48:08.860450+0000","last_peered":"2024-09-13T14:48:08.860450+0000","last_clean":"2024-09-13T14:48:08.860450+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:48:08.860450+0000","last_undegraded":"2024-09-13T14:48:08.860450+0000","last_fullsized":"2024-09-13T14:48:08.860450+0000","mapping_epoch":303,"log_start":"306'1200","ondisk_log_start":"306'1200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":331,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421040+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:47:55.421040+0000","last_peered":"2024-09-13T14:47:55.421040+0000","last_clean":"2024-09-13T14:47:55.421040+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:47:55.421040+0000","last_undegraded":"2024-09-13T14:47:55.421040+0000","last_fullsized":"2024-09-13T14:47:55.421040+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":346,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.415421+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:47:55.415421+0000","last_peered":"2024-09-13T14:47:55.415421+0000","last_clean":"2024-09-13T14:47:55.415421+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:47:55.415421+0000","last_undegraded":"2024-09-13T14:47:55.415421+0000","last_fullsized":"2024-09-13T14:47:55.415421+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"365'9627","reported_seq":9765,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:48:09.874315+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:48:09.874315+0000","last_peered":"2024-09-13T14:48:09.874315+0000","last_clean":"2024-09-13T14:48:09.874315+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:48:09.874315+0000","last_undegraded":"2024-09-13T14:48:09.874315+0000","last_fullsized":"2024-09-13T14:48:09.874315+0000","mapping_epoch":303,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7527,"log_dups_size":0,"ondisk_log_size":7527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125763584,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9627,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":294,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414738+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:47:55.414738+0000","last_peered":"2024-09-13T14:47:55.414738+0000","last_clean":"2024-09-13T14:47:55.414738+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:47:55.414738+0000","last_undegraded":"2024-09-13T14:47:55.414738+0000","last_fullsized":"2024-09-13T14:47:55.414738+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":626,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420389+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:47:55.420389+0000","last_peered":"2024-09-13T14:47:55.420389+0000","last_clean":"2024-09-13T14:47:55.420389+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:47:55.420389+0000","last_undegraded":"2024-09-13T14:47:55.420389+0000","last_fullsized":"2024-09-13T14:47:55.420389+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"366'9589","reported_seq":9727,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:10.683569+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:48:10.683569+0000","last_peered":"2024-09-13T14:48:10.683569+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:48:10.683569+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'2600","ondisk_log_start":"307'2600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122748928,"num_objects":1873,"num_object_clones":0,"num_object_copies":5619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9589,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-3760}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":477,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416782+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:47:55.416782+0000","last_peered":"2024-09-13T14:47:55.416782+0000","last_clean":"2024-09-13T14:47:55.416782+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:47:55.416782+0000","last_undegraded":"2024-09-13T14:47:55.416782+0000","last_fullsized":"2024-09-13T14:47:55.416782+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":626,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361933+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:48:10.361933+0000","last_peered":"2024-09-13T14:48:10.361933+0000","last_clean":"2024-09-13T14:48:10.361933+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:48:10.361933+0000","last_undegraded":"2024-09-13T14:48:10.361933+0000","last_fullsized":"2024-09-13T14:48:10.361933+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422899+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:47:55.422899+0000","last_peered":"2024-09-13T14:47:55.422899+0000","last_clean":"2024-09-13T14:47:55.422899+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:47:55.422899+0000","last_undegraded":"2024-09-13T14:47:55.422899+0000","last_fullsized":"2024-09-13T14:47:55.422899+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"365'9269","reported_seq":9408,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.034361+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:48:10.034361+0000","last_peered":"2024-09-13T14:48:10.034361+0000","last_clean":"2024-09-13T14:48:10.034361+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:48:10.034361+0000","last_undegraded":"2024-09-13T14:48:10.034361+0000","last_fullsized":"2024-09-13T14:48:10.034361+0000","mapping_epoch":303,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119603200,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9269,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":181,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414347+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:47:55.414347+0000","last_peered":"2024-09-13T14:47:55.414347+0000","last_clean":"2024-09-13T14:47:55.414347+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:47:55.414347+0000","last_undegraded":"2024-09-13T14:47:55.414347+0000","last_fullsized":"2024-09-13T14:47:55.414347+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":367,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.415296+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:47:55.415296+0000","last_peered":"2024-09-13T14:47:55.415296+0000","last_clean":"2024-09-13T14:47:55.415296+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:47:55.415296+0000","last_undegraded":"2024-09-13T14:47:55.415296+0000","last_fullsized":"2024-09-13T14:47:55.415296+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423241+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:47:55.423241+0000","last_peered":"2024-09-13T14:47:55.423241+0000","last_clean":"2024-09-13T14:47:55.423241+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:47:55.423241+0000","last_undegraded":"2024-09-13T14:47:55.423241+0000","last_fullsized":"2024-09-13T14:47:55.423241+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"366'9537","reported_seq":20757,"reported_epoch":366,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:12.614250+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:48:12.614250+0000","last_peered":"2024-09-13T14:48:12.614250+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:48:12.614250+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'2200","ondisk_log_start":"307'2200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7337,"log_dups_size":0,"ondisk_log_size":7337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":820,"num_objects_missing":820,"num_objects_degraded":1195,"num_objects_misplaced":1772,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3831,"num_bytes_recovered":251068416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":820},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422610+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:47:55.422610+0000","last_peered":"2024-09-13T14:47:55.422610+0000","last_clean":"2024-09-13T14:47:55.422610+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:47:55.422610+0000","last_undegraded":"2024-09-13T14:47:55.422610+0000","last_fullsized":"2024-09-13T14:47:55.422610+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423135+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:47:55.423135+0000","last_peered":"2024-09-13T14:47:55.423135+0000","last_clean":"2024-09-13T14:47:55.423135+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:47:55.423135+0000","last_undegraded":"2024-09-13T14:47:55.423135+0000","last_fullsized":"2024-09-13T14:47:55.423135+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"365'9270","reported_seq":9409,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:10.060806+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:48:10.060806+0000","last_peered":"2024-09-13T14:48:10.060806+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:48:10.060806+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'1700","ondisk_log_start":"307'1700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9270,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-3578}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":174,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414397+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:47:55.414397+0000","last_peered":"2024-09-13T14:47:55.414397+0000","last_clean":"2024-09-13T14:47:55.414397+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:47:55.414397+0000","last_undegraded":"2024-09-13T14:47:55.414397+0000","last_fullsized":"2024-09-13T14:47:55.414397+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":619,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.369585+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:48:10.369585+0000","last_peered":"2024-09-13T14:48:10.369585+0000","last_clean":"2024-09-13T14:48:10.369585+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:48:10.369585+0000","last_undegraded":"2024-09-13T14:48:10.369585+0000","last_fullsized":"2024-09-13T14:48:10.369585+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":578,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423580+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:47:55.423580+0000","last_peered":"2024-09-13T14:47:55.423580+0000","last_clean":"2024-09-13T14:47:55.423580+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:47:55.423580+0000","last_undegraded":"2024-09-13T14:47:55.423580+0000","last_fullsized":"2024-09-13T14:47:55.423580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"366'9413","reported_seq":9491,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:12.517003+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:48:12.517003+0000","last_peered":"2024-09-13T14:48:12.517003+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:48:12.517003+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118423552,"num_objects":1807,"num_object_clones":0,"num_object_copies":5421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9413,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-3689}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":468,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416921+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:47:55.416921+0000","last_peered":"2024-09-13T14:47:55.416921+0000","last_clean":"2024-09-13T14:47:55.416921+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:47:55.416921+0000","last_undegraded":"2024-09-13T14:47:55.416921+0000","last_fullsized":"2024-09-13T14:47:55.416921+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":890,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414540+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:47:55.414540+0000","last_peered":"2024-09-13T14:47:55.414540+0000","last_clean":"2024-09-13T14:47:55.414540+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:47:55.414540+0000","last_undegraded":"2024-09-13T14:47:55.414540+0000","last_fullsized":"2024-09-13T14:47:55.414540+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":572,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361086+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:48:10.361086+0000","last_peered":"2024-09-13T14:48:10.361086+0000","last_clean":"2024-09-13T14:48:10.361086+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:48:10.361086+0000","last_undegraded":"2024-09-13T14:48:10.361086+0000","last_fullsized":"2024-09-13T14:48:10.361086+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419145+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:47:55.419145+0000","last_peered":"2024-09-13T14:47:55.419145+0000","last_clean":"2024-09-13T14:47:55.419145+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:47:55.419145+0000","last_undegraded":"2024-09-13T14:47:55.419145+0000","last_fullsized":"2024-09-13T14:47:55.419145+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"365'9670","reported_seq":9806,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:10.085039+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:48:10.085039+0000","last_peered":"2024-09-13T14:48:10.085039+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:48:10.085039+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9670,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-3802}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":424,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423361+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:47:55.423361+0000","last_peered":"2024-09-13T14:47:55.423361+0000","last_clean":"2024-09-13T14:47:55.423361+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:47:55.423361+0000","last_undegraded":"2024-09-13T14:47:55.423361+0000","last_fullsized":"2024-09-13T14:47:55.423361+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":756,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416867+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:47:55.416867+0000","last_peered":"2024-09-13T14:47:55.416867+0000","last_clean":"2024-09-13T14:47:55.416867+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:47:55.416867+0000","last_undegraded":"2024-09-13T14:47:55.416867+0000","last_fullsized":"2024-09-13T14:47:55.416867+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":619,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419980+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:47:55.419980+0000","last_peered":"2024-09-13T14:47:55.419980+0000","last_clean":"2024-09-13T14:47:55.419980+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:47:55.419980+0000","last_undegraded":"2024-09-13T14:47:55.419980+0000","last_fullsized":"2024-09-13T14:47:55.419980+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":624,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421093+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:47:55.421093+0000","last_peered":"2024-09-13T14:47:55.421093+0000","last_clean":"2024-09-13T14:47:55.421093+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:47:55.421093+0000","last_undegraded":"2024-09-13T14:47:55.421093+0000","last_fullsized":"2024-09-13T14:47:55.421093+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"366'9680","reported_seq":19361,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.613609+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:48:10.613609+0000","last_peered":"2024-09-13T14:48:10.613609+0000","last_clean":"2024-09-13T14:48:10.613609+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:48:10.613609+0000","last_undegraded":"2024-09-13T14:48:10.613609+0000","last_fullsized":"2024-09-13T14:48:10.613609+0000","mapping_epoch":330,"log_start":"307'2700","ondisk_log_start":"307'2700","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130023424,"num_objects":1984,"num_object_clones":0,"num_object_copies":5952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9680,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":497,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.417104+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:47:55.417104+0000","last_peered":"2024-09-13T14:47:55.417104+0000","last_clean":"2024-09-13T14:47:55.417104+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:47:55.417104+0000","last_undegraded":"2024-09-13T14:47:55.417104+0000","last_fullsized":"2024-09-13T14:47:55.417104+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":590,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361171+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:48:10.361171+0000","last_peered":"2024-09-13T14:48:10.361171+0000","last_clean":"2024-09-13T14:48:10.361171+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:48:10.361171+0000","last_undegraded":"2024-09-13T14:48:10.361171+0000","last_fullsized":"2024-09-13T14:48:10.361171+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":556,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423255+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:47:55.423255+0000","last_peered":"2024-09-13T14:47:55.423255+0000","last_clean":"2024-09-13T14:47:55.423255+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:47:55.423255+0000","last_undegraded":"2024-09-13T14:47:55.423255+0000","last_fullsized":"2024-09-13T14:47:55.423255+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414581+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:47:55.414581+0000","last_peered":"2024-09-13T14:47:55.414581+0000","last_clean":"2024-09-13T14:47:55.414581+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:47:55.414581+0000","last_undegraded":"2024-09-13T14:47:55.414581+0000","last_fullsized":"2024-09-13T14:47:55.414581+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"366'9329","reported_seq":9468,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:10.588715+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:48:10.588715+0000","last_peered":"2024-09-13T14:48:10.588715+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:48:10.588715+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124583936,"num_objects":1901,"num_object_clones":0,"num_object_copies":5703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9329,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-3608}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":174,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414261+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:47:55.414261+0000","last_peered":"2024-09-13T14:47:55.414261+0000","last_clean":"2024-09-13T14:47:55.414261+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:47:55.414261+0000","last_undegraded":"2024-09-13T14:47:55.414261+0000","last_fullsized":"2024-09-13T14:47:55.414261+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":981,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414425+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:47:55.414425+0000","last_peered":"2024-09-13T14:47:55.414425+0000","last_clean":"2024-09-13T14:47:55.414425+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:47:55.414425+0000","last_undegraded":"2024-09-13T14:47:55.414425+0000","last_fullsized":"2024-09-13T14:47:55.414425+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422976+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:47:55.422976+0000","last_peered":"2024-09-13T14:47:55.422976+0000","last_clean":"2024-09-13T14:47:55.422976+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:47:55.422976+0000","last_undegraded":"2024-09-13T14:47:55.422976+0000","last_fullsized":"2024-09-13T14:47:55.422976+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":577,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422488+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:47:55.422488+0000","last_peered":"2024-09-13T14:47:55.422488+0000","last_clean":"2024-09-13T14:47:55.422488+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:47:55.422488+0000","last_undegraded":"2024-09-13T14:47:55.422488+0000","last_fullsized":"2024-09-13T14:47:55.422488+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"365'9427","reported_seq":18845,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:48:08.890716+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:48:08.890716+0000","last_peered":"2024-09-13T14:48:08.890716+0000","last_clean":"2024-09-13T14:48:08.890716+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:48:08.890716+0000","last_undegraded":"2024-09-13T14:48:08.890716+0000","last_fullsized":"2024-09-13T14:48:08.890716+0000","mapping_epoch":337,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9427,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.415941+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:47:55.415941+0000","last_peered":"2024-09-13T14:47:55.415941+0000","last_clean":"2024-09-13T14:47:55.415941+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:47:55.415941+0000","last_undegraded":"2024-09-13T14:47:55.415941+0000","last_fullsized":"2024-09-13T14:47:55.415941+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414676+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:47:55.414676+0000","last_peered":"2024-09-13T14:47:55.414676+0000","last_clean":"2024-09-13T14:47:55.414676+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:47:55.414676+0000","last_undegraded":"2024-09-13T14:47:55.414676+0000","last_fullsized":"2024-09-13T14:47:55.414676+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1156,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422681+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:47:55.422681+0000","last_peered":"2024-09-13T14:47:55.422681+0000","last_clean":"2024-09-13T14:47:55.422681+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:47:55.422681+0000","last_undegraded":"2024-09-13T14:47:55.422681+0000","last_fullsized":"2024-09-13T14:47:55.422681+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":558,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424304+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:47:55.424304+0000","last_peered":"2024-09-13T14:47:55.424304+0000","last_clean":"2024-09-13T14:47:55.424304+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:47:55.424304+0000","last_undegraded":"2024-09-13T14:47:55.424304+0000","last_fullsized":"2024-09-13T14:47:55.424304+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"366'9318","reported_seq":9458,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.500783+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:48:10.500783+0000","last_peered":"2024-09-13T14:48:10.500783+0000","last_clean":"2024-09-13T14:48:10.500783+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:48:10.500783+0000","last_undegraded":"2024-09-13T14:48:10.500783+0000","last_fullsized":"2024-09-13T14:48:10.500783+0000","mapping_epoch":303,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123600896,"num_objects":1886,"num_object_clones":0,"num_object_copies":5658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9318,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.415504+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:47:55.415504+0000","last_peered":"2024-09-13T14:47:55.415504+0000","last_clean":"2024-09-13T14:47:55.415504+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:47:55.415504+0000","last_undegraded":"2024-09-13T14:47:55.415504+0000","last_fullsized":"2024-09-13T14:47:55.415504+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":567,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422590+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:47:55.422590+0000","last_peered":"2024-09-13T14:47:55.422590+0000","last_clean":"2024-09-13T14:47:55.422590+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:47:55.422590+0000","last_undegraded":"2024-09-13T14:47:55.422590+0000","last_fullsized":"2024-09-13T14:47:55.422590+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":760,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421042+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:47:55.421042+0000","last_peered":"2024-09-13T14:47:55.421042+0000","last_clean":"2024-09-13T14:47:55.421042+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:47:55.421042+0000","last_undegraded":"2024-09-13T14:47:55.421042+0000","last_fullsized":"2024-09-13T14:47:55.421042+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":602,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420315+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:47:55.420315+0000","last_peered":"2024-09-13T14:47:55.420315+0000","last_clean":"2024-09-13T14:47:55.420315+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:47:55.420315+0000","last_undegraded":"2024-09-13T14:47:55.420315+0000","last_fullsized":"2024-09-13T14:47:55.420315+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"365'9724","reported_seq":9862,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:48:08.914009+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:48:08.914009+0000","last_peered":"2024-09-13T14:48:08.914009+0000","last_clean":"2024-09-13T14:48:08.914009+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:48:08.914009+0000","last_undegraded":"2024-09-13T14:48:08.914009+0000","last_fullsized":"2024-09-13T14:48:08.914009+0000","mapping_epoch":303,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8624,"log_dups_size":0,"ondisk_log_size":8624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843776,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9724,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423066+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:47:55.423066+0000","last_peered":"2024-09-13T14:47:55.423066+0000","last_clean":"2024-09-13T14:47:55.423066+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:47:55.423066+0000","last_undegraded":"2024-09-13T14:47:55.423066+0000","last_fullsized":"2024-09-13T14:47:55.423066+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":605,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361536+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:48:10.361536+0000","last_peered":"2024-09-13T14:48:10.361536+0000","last_clean":"2024-09-13T14:48:10.361536+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:48:10.361536+0000","last_undegraded":"2024-09-13T14:48:10.361536+0000","last_fullsized":"2024-09-13T14:48:10.361536+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":943,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422711+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:47:55.422711+0000","last_peered":"2024-09-13T14:47:55.422711+0000","last_clean":"2024-09-13T14:47:55.422711+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:47:55.422711+0000","last_undegraded":"2024-09-13T14:47:55.422711+0000","last_fullsized":"2024-09-13T14:47:55.422711+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":603,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.369355+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:48:10.369355+0000","last_peered":"2024-09-13T14:48:10.369355+0000","last_clean":"2024-09-13T14:48:10.369355+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:48:10.369355+0000","last_undegraded":"2024-09-13T14:48:10.369355+0000","last_fullsized":"2024-09-13T14:48:10.369355+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"366'9702","reported_seq":21079,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:12.723751+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:48:12.723751+0000","last_peered":"2024-09-13T14:48:12.723751+0000","last_clean":"2024-09-13T14:48:12.723751+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:48:12.723751+0000","last_undegraded":"2024-09-13T14:48:12.723751+0000","last_fullsized":"2024-09-13T14:48:12.723751+0000","mapping_epoch":332,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9102,"log_dups_size":0,"ondisk_log_size":9102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118685720,"num_objects":1812,"num_object_clones":0,"num_object_copies":5436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1812,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9702,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":456,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.364662+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:48:10.364662+0000","last_peered":"2024-09-13T14:48:10.364662+0000","last_clean":"2024-09-13T14:48:10.364662+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:48:10.364662+0000","last_undegraded":"2024-09-13T14:48:10.364662+0000","last_fullsized":"2024-09-13T14:48:10.364662+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":638,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420841+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:47:55.420841+0000","last_peered":"2024-09-13T14:47:55.420841+0000","last_clean":"2024-09-13T14:47:55.420841+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:47:55.420841+0000","last_undegraded":"2024-09-13T14:47:55.420841+0000","last_fullsized":"2024-09-13T14:47:55.420841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":784,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416066+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:47:55.416066+0000","last_peered":"2024-09-13T14:47:55.416066+0000","last_clean":"2024-09-13T14:47:55.416066+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:47:55.416066+0000","last_undegraded":"2024-09-13T14:47:55.416066+0000","last_fullsized":"2024-09-13T14:47:55.416066+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422686+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:47:55.422686+0000","last_peered":"2024-09-13T14:47:55.422686+0000","last_clean":"2024-09-13T14:47:55.422686+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:47:55.422686+0000","last_undegraded":"2024-09-13T14:47:55.422686+0000","last_fullsized":"2024-09-13T14:47:55.422686+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.423039+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:47:55.423039+0000","last_peered":"2024-09-13T14:47:55.423039+0000","last_clean":"2024-09-13T14:47:55.423039+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:47:55.423039+0000","last_undegraded":"2024-09-13T14:47:55.423039+0000","last_fullsized":"2024-09-13T14:47:55.423039+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":75,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422764+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:47:55.422764+0000","last_peered":"2024-09-13T14:47:55.422764+0000","last_clean":"2024-09-13T14:47:55.422764+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:47:55.422764+0000","last_undegraded":"2024-09-13T14:47:55.422764+0000","last_fullsized":"2024-09-13T14:47:55.422764+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":554,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.424157+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:47:55.424157+0000","last_peered":"2024-09-13T14:47:55.424157+0000","last_clean":"2024-09-13T14:47:55.424157+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:47:55.424157+0000","last_undegraded":"2024-09-13T14:47:55.424157+0000","last_fullsized":"2024-09-13T14:47:55.424157+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420958+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:47:55.420958+0000","last_peered":"2024-09-13T14:47:55.420958+0000","last_clean":"2024-09-13T14:47:55.420958+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:47:55.420958+0000","last_undegraded":"2024-09-13T14:47:55.420958+0000","last_fullsized":"2024-09-13T14:47:55.420958+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":331,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414663+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:47:55.414663+0000","last_peered":"2024-09-13T14:47:55.414663+0000","last_clean":"2024-09-13T14:47:55.414663+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:47:55.414663+0000","last_undegraded":"2024-09-13T14:47:55.414663+0000","last_fullsized":"2024-09-13T14:47:55.414663+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":561,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360889+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:48:10.360889+0000","last_peered":"2024-09-13T14:48:10.360889+0000","last_clean":"2024-09-13T14:48:10.360889+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:48:10.360889+0000","last_undegraded":"2024-09-13T14:48:10.360889+0000","last_fullsized":"2024-09-13T14:48:10.360889+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":610,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419578+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:47:55.419578+0000","last_peered":"2024-09-13T14:47:55.419578+0000","last_clean":"2024-09-13T14:47:55.419578+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:47:55.419578+0000","last_undegraded":"2024-09-13T14:47:55.419578+0000","last_fullsized":"2024-09-13T14:47:55.419578+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":617,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.369244+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:48:10.369244+0000","last_peered":"2024-09-13T14:48:10.369244+0000","last_clean":"2024-09-13T14:48:10.369244+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:48:10.369244+0000","last_undegraded":"2024-09-13T14:48:10.369244+0000","last_fullsized":"2024-09-13T14:48:10.369244+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":363,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.416003+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:47:55.416003+0000","last_peered":"2024-09-13T14:47:55.416003+0000","last_clean":"2024-09-13T14:47:55.416003+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:47:55.416003+0000","last_undegraded":"2024-09-13T14:47:55.416003+0000","last_fullsized":"2024-09-13T14:47:55.416003+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":635,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.369525+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:48:10.369525+0000","last_peered":"2024-09-13T14:48:10.369525+0000","last_clean":"2024-09-13T14:48:10.369525+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:48:10.369525+0000","last_undegraded":"2024-09-13T14:48:10.369525+0000","last_fullsized":"2024-09-13T14:48:10.369525+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":603,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.421021+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:47:55.421021+0000","last_peered":"2024-09-13T14:47:55.421021+0000","last_clean":"2024-09-13T14:47:55.421021+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:47:55.421021+0000","last_undegraded":"2024-09-13T14:47:55.421021+0000","last_fullsized":"2024-09-13T14:47:55.421021+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":579,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422323+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:47:55.422323+0000","last_peered":"2024-09-13T14:47:55.422323+0000","last_clean":"2024-09-13T14:47:55.422323+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:47:55.422323+0000","last_undegraded":"2024-09-13T14:47:55.422323+0000","last_fullsized":"2024-09-13T14:47:55.422323+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":630,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361249+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:48:10.361249+0000","last_peered":"2024-09-13T14:48:10.361249+0000","last_clean":"2024-09-13T14:48:10.361249+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:48:10.361249+0000","last_undegraded":"2024-09-13T14:48:10.361249+0000","last_fullsized":"2024-09-13T14:48:10.361249+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":64,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.439153+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:47:55.439153+0000","last_peered":"2024-09-13T14:47:55.439153+0000","last_clean":"2024-09-13T14:47:55.439153+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:47:55.439153+0000","last_undegraded":"2024-09-13T14:47:55.439153+0000","last_fullsized":"2024-09-13T14:47:55.439153+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":525,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422804+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:47:55.422804+0000","last_peered":"2024-09-13T14:47:55.422804+0000","last_clean":"2024-09-13T14:47:55.422804+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:47:55.422804+0000","last_undegraded":"2024-09-13T14:47:55.422804+0000","last_fullsized":"2024-09-13T14:47:55.422804+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":606,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.420220+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:47:55.420220+0000","last_peered":"2024-09-13T14:47:55.420220+0000","last_clean":"2024-09-13T14:47:55.420220+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:47:55.420220+0000","last_undegraded":"2024-09-13T14:47:55.420220+0000","last_fullsized":"2024-09-13T14:47:55.420220+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422899+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:47:55.422899+0000","last_peered":"2024-09-13T14:47:55.422899+0000","last_clean":"2024-09-13T14:47:55.422899+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:47:55.422899+0000","last_undegraded":"2024-09-13T14:47:55.422899+0000","last_fullsized":"2024-09-13T14:47:55.422899+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":331,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414777+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:47:55.414777+0000","last_peered":"2024-09-13T14:47:55.414777+0000","last_clean":"2024-09-13T14:47:55.414777+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:47:55.414777+0000","last_undegraded":"2024-09-13T14:47:55.414777+0000","last_fullsized":"2024-09-13T14:47:55.414777+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.439098+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:47:55.439098+0000","last_peered":"2024-09-13T14:47:55.439098+0000","last_clean":"2024-09-13T14:47:55.439098+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:47:55.439098+0000","last_undegraded":"2024-09-13T14:47:55.439098+0000","last_fullsized":"2024-09-13T14:47:55.439098+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":297,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.414755+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:47:55.414755+0000","last_peered":"2024-09-13T14:47:55.414755+0000","last_clean":"2024-09-13T14:47:55.414755+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:47:55.414755+0000","last_undegraded":"2024-09-13T14:47:55.414755+0000","last_fullsized":"2024-09-13T14:47:55.414755+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":627,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361311+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:48:10.361311+0000","last_peered":"2024-09-13T14:48:10.361311+0000","last_clean":"2024-09-13T14:48:10.361311+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:48:10.361311+0000","last_undegraded":"2024-09-13T14:48:10.361311+0000","last_fullsized":"2024-09-13T14:48:10.361311+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":614,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361632+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:48:10.361632+0000","last_peered":"2024-09-13T14:48:10.361632+0000","last_clean":"2024-09-13T14:48:10.361632+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:48:10.361632+0000","last_undegraded":"2024-09-13T14:48:10.361632+0000","last_fullsized":"2024-09-13T14:48:10.361632+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":65,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.422560+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:47:55.422560+0000","last_peered":"2024-09-13T14:47:55.422560+0000","last_clean":"2024-09-13T14:47:55.422560+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:47:55.422560+0000","last_undegraded":"2024-09-13T14:47:55.422560+0000","last_fullsized":"2024-09-13T14:47:55.422560+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":616,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-09-13T14:47:55.419487+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:47:55.419487+0000","last_peered":"2024-09-13T14:47:55.419487+0000","last_clean":"2024-09-13T14:47:55.419487+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:47:55.419487+0000","last_undegraded":"2024-09-13T14:47:55.419487+0000","last_fullsized":"2024-09-13T14:47:55.419487+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1978531864,"num_objects":30191,"num_object_clones":0,"num_object_copies":90573,"num_objects_missing_on_primary":820,"num_objects_missing":820,"num_objects_degraded":31524,"num_objects_misplaced":1772,"num_objects_unfound":0,"num_objects_dirty":30191,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152487,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27458,"num_bytes_recovered":1739063296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6036992000,"data_stored":6036979784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125087,"ondisk_log_size":125087,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397945,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507676,"kb_used_data":607508,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":92237476,"statfs":{"total":95995035648,"available":94451175424,"internally_reserved":0,"allocated":622088192,"data_stored":620123231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594173,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553484,"kb_used_data":955264,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":92191668,"statfs":{"total":95995035648,"available":94404268032,"internally_reserved":0,"allocated":978190336,"data_stored":976245129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665627,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492232,"kb_used_data":168000,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93252920,"statfs":{"total":95995035648,"available":95490990080,"internally_reserved":0,"allocated":172032000,"data_stored":170083124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986629,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1402712,"kb_used_data":502860,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92342440,"statfs":{"total":95995035648,"available":94558658560,"internally_reserved":0,"allocated":514928640,"data_stored":512971777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,3,4,6,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697188,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847476,"kb_used_data":1101804,"kb_used_omap":4,"kb_used_meta":745659,"kb_avail":91897676,"statfs":{"total":95995035648,"available":94103220224,"internally_reserved":0,"allocated":1128247296,"data_stored":1126310420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":763554891},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860715,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646544,"kb_used_data":1021124,"kb_used_omap":8,"kb_used_meta":625399,"kb_avail":92098608,"statfs":{"total":95995035648,"available":94308974592,"internally_reserved":0,"allocated":1045630976,"data_stored":1043678599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":640409420},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020386,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183664,"kb_used_data":862056,"kb_used_omap":9,"kb_used_meta":321590,"kb_avail":92561488,"statfs":{"total":95995035648,"available":94782963712,"internally_reserved":0,"allocated":882745344,"data_stored":880811941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":329308997},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187776,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1376752,"kb_used_data":744680,"kb_used_omap":7,"kb_used_meta":632056,"kb_avail":92368400,"statfs":{"total":95995035648,"available":94585241600,"internally_reserved":0,"allocated":762552320,"data_stored":760619183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":647225675},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":752025600,"data_stored":752025600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":870580224,"data_stored":870580224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1034948608,"data_stored":1034944536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1120010240,"data_stored":1120010240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":508690432,"data_stored":508690432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":166531072,"data_stored":166527000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":972750848,"data_stored":972750848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":611454976,"data_stored":611450904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:48:17.142 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.164 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.266 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:17.514 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:17.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:17.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:17.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:48:17.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:17.646 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:17] "GET /metrics HTTP/1.1" 200 36700 "" "Prometheus/2.51.0" 2024-09-13T14:48:17.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:17.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.813 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.916 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:17.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:18.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:18.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:18.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:18.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:18.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:18.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.333 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:18.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:18 smithi017 ceph-mon[103001]: pgmap v1190: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31370/87777 objects degraded (35.738%); 1772/87777 objects misplaced (2.019%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:48:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:18 smithi017 ceph-mon[107556]: pgmap v1190: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31370/87777 objects degraded (35.738%); 1772/87777 objects misplaced (2.019%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:48:18.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:18 smithi189 ceph-mon[90992]: pgmap v1190: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31370/87777 objects degraded (35.738%); 1772/87777 objects misplaced (2.019%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:48:18.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:18.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:18.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.067 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:19.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:19.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:19.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:19.218 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:19.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 31370/87777 objects degraded (35.738%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 31370/87777 objects degraded (35.738%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:19.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 31370/87777 objects degraded (35.738%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:19.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:19.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:19.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:19.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:19.857 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:19.959 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:20.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:48:20.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.091 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:20.109 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:20.109 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:20.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:20.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:20.338 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:48:20.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:20 smithi017 ceph-mon[103001]: pgmap v1191: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31293/86223 objects degraded (36.293%); 1694/86223 objects misplaced (1.965%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:20.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:20.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:20 smithi017 ceph-mon[107556]: pgmap v1191: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31293/86223 objects degraded (36.293%); 1694/86223 objects misplaced (1.965%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:20.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:20.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:20.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:20 smithi189 ceph-mon[90992]: pgmap v1191: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 31293/86223 objects degraded (36.293%); 1694/86223 objects misplaced (1.965%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:20.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:20.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:20.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:20.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:20.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:21.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:21.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:21.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:21.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:21.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:48:21.367 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:21.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:21.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:21.516 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:21.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:21.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:21.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[103001]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:48:21.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:21.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:21.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:21 smithi017 ceph-mon[107556]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:48:21.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:21.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:21 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2533154033' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:21.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:21 smithi189 ceph-mon[90992]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:48:21.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:21.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:21.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:21.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:22.012 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.042 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:22.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:22.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:22.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:22.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:22.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:22.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.643 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:22.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:22.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:22 smithi017 ceph-mon[103001]: pgmap v1193: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 31293/84267 objects degraded (37.136%); 1694/84267 objects misplaced (2.010%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:48:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:22 smithi017 ceph-mon[107556]: pgmap v1193: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 31293/84267 objects degraded (37.136%); 1694/84267 objects misplaced (2.010%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:48:22.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:22 smithi189 ceph-mon[90992]: pgmap v1193: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 243 op/s; 31293/84267 objects degraded (37.136%); 1694/84267 objects misplaced (2.010%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T14:48:22.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:22.918 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:23.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:23.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:23.147 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:23.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:23.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:23.298 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:23.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:23.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:23.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:23.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:23.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:48:23.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:23.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:23.775 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:23.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:23.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:24.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:24.089 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:48:24.089 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:24.117 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:48:24.117 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:24.140 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:48:24.140 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:24.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:24.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:48:24.160 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:24.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:24.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:24.180 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:48:24.180 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:24.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:48:24.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:24.244 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:48:24.244 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:24.267 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:48:24.267 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:24.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.294 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:24.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[103001]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 31054/83061 objects degraded (37.387%); 1615/83061 objects misplaced (1.944%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:24.342 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 31054/83061 objects degraded (37.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:24.342 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3387054310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:24.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:24.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:24.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:24.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:24.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[107556]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 31054/83061 objects degraded (37.387%); 1615/83061 objects misplaced (1.944%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:24.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 31054/83061 objects degraded (37.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:24.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:24 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3387054310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:24.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:24.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:24 smithi189 ceph-mon[90992]: pgmap v1194: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 31054/83061 objects degraded (37.387%); 1615/83061 objects misplaced (1.944%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 31054/83061 objects degraded (37.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:24.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:24 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3387054310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:24.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:24.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:24.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:24.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:25.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:25.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:25.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:25.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:25.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:25.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:25.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:25.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:25.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:25.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:25.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:25.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:25.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:26.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:26.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:26.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.484 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:26.535 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:26.536 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:26.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:26.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:26.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:26.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:26 smithi017 ceph-mon[103001]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 31054/81315 objects degraded (38.190%); 1615/81315 objects misplaced (1.986%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:26 smithi017 ceph-mon[107556]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 31054/81315 objects degraded (38.190%); 1615/81315 objects misplaced (1.986%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:26.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:26 smithi189 ceph-mon[90992]: pgmap v1195: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 31054/81315 objects degraded (38.190%); 1615/81315 objects misplaced (1.986%); 1.9 MiB/s, 31 objects/s recovering 2024-09-13T14:48:27.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:27.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:27.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:27.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:27.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:27.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.602 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:27] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:48:27.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:27.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:27.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:27.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:27.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:27.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:28.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:28.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:28.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:28.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[107556]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 30870/79917 objects degraded (38.628%); 1615/79917 objects misplaced (2.021%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:28.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:28 smithi017 ceph-mon[103001]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 30870/79917 objects degraded (38.628%); 1615/79917 objects misplaced (2.021%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:28.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:28.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:28 smithi189 ceph-mon[90992]: pgmap v1196: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 30870/79917 objects degraded (38.628%); 1615/79917 objects misplaced (2.021%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:48:28.506 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:28.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:28.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:28.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:28.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:28.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:28.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:29.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:29.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:29.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:29.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:29.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.360 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:29.390 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:29.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 30870/79917 objects degraded (38.628%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:29.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 30870/79917 objects degraded (38.628%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:29.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 30870/79917 objects degraded (38.628%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:29.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.642 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:29.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:29.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:29.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:29.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:30.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:30.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:30.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:30.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:30.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:30.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:30 smithi189 ceph-mon[90992]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 30787/78165 objects degraded (39.387%); 1529/78165 objects misplaced (1.956%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:30.407 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:30.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:30 smithi017 ceph-mon[103001]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 30787/78165 objects degraded (39.387%); 1529/78165 objects misplaced (1.956%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:30 smithi017 ceph-mon[107556]: pgmap v1197: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 30787/78165 objects degraded (39.387%); 1529/78165 objects misplaced (1.956%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:48:30.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:30.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:30.896 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:30.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:30.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.022 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:31.024 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:48:31.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:31.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:31.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:31.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:31 smithi189 ceph-mon[90992]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:31.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:31.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:31.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:31 smithi017 ceph-mon[103001]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:31.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:31 smithi017 ceph-mon[107556]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:31.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:31.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:31.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:31.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:31.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:31.794 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1197,"stamp":"2024-09-13T14:48:29.275341+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1710328502,"num_objects":26055,"num_object_clones":0,"num_object_copies":78165,"num_objects_missing_on_primary":577,"num_objects_missing":577,"num_objects_degraded":30787,"num_objects_misplaced":1529,"num_objects_unfound":0,"num_objects_dirty":26055,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":157598,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28005,"num_bytes_recovered":1771341981,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127583,"ondisk_log_size":127583,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10375248,"kb_used_data":5318416,"kb_used_omap":44,"kb_used_meta":5056723,"kb_avail":739585968,"statfs":{"total":767960285184,"available":757336031232,"internally_reserved":0,"allocated":5446057984,"data_stored":5430399967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5178084599},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,2,0,2,2,2,6,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-176029696,"num_objects":-2686,"num_object_clones":0,"num_object_copies":-8058,"num_objects_missing_on_primary":-166,"num_objects_missing":-166,"num_objects_degraded":-506,"num_objects_misplaced":-165,"num_objects_unfound":0,"num_objects_dirty":-2686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2686,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21954560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002900"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":619,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.342062+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:48:20.342062+0000","last_peered":"2024-09-13T14:48:20.342062+0000","last_clean":"2024-09-13T14:48:20.342062+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:48:20.342062+0000","last_undegraded":"2024-09-13T14:48:20.342062+0000","last_fullsized":"2024-09-13T14:48:20.342062+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":365,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360830+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:48:10.360830+0000","last_peered":"2024-09-13T14:48:10.360830+0000","last_clean":"2024-09-13T14:48:10.360830+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:48:10.360830+0000","last_undegraded":"2024-09-13T14:48:10.360830+0000","last_fullsized":"2024-09-13T14:48:10.360830+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":294,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350815+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:48:10.350815+0000","last_peered":"2024-09-13T14:48:10.350815+0000","last_clean":"2024-09-13T14:48:10.350815+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:48:10.350815+0000","last_undegraded":"2024-09-13T14:48:10.350815+0000","last_fullsized":"2024-09-13T14:48:10.350815+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":604,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341102+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:48:20.341102+0000","last_peered":"2024-09-13T14:48:20.341102+0000","last_clean":"2024-09-13T14:48:20.341102+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:48:20.341102+0000","last_undegraded":"2024-09-13T14:48:20.341102+0000","last_fullsized":"2024-09-13T14:48:20.341102+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":605,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341477+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:48:20.341477+0000","last_peered":"2024-09-13T14:48:20.341477+0000","last_clean":"2024-09-13T14:48:20.341477+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:48:20.341477+0000","last_undegraded":"2024-09-13T14:48:20.341477+0000","last_fullsized":"2024-09-13T14:48:20.341477+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":334,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360631+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:48:10.360631+0000","last_peered":"2024-09-13T14:48:10.360631+0000","last_clean":"2024-09-13T14:48:10.360631+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:48:10.360631+0000","last_undegraded":"2024-09-13T14:48:10.360631+0000","last_fullsized":"2024-09-13T14:48:10.360631+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":619,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.343307+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:48:20.343307+0000","last_peered":"2024-09-13T14:48:20.343307+0000","last_clean":"2024-09-13T14:48:20.343307+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:48:20.343307+0000","last_undegraded":"2024-09-13T14:48:20.343307+0000","last_fullsized":"2024-09-13T14:48:20.343307+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":356,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345234+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:48:10.345234+0000","last_peered":"2024-09-13T14:48:10.345234+0000","last_clean":"2024-09-13T14:48:10.345234+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:48:10.345234+0000","last_undegraded":"2024-09-13T14:48:10.345234+0000","last_fullsized":"2024-09-13T14:48:10.345234+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"367'9721","reported_seq":22088,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:27.089518+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:48:27.089518+0000","last_peered":"2024-09-13T14:48:27.089518+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:48:27.089518+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9721,"log_dups_size":0,"ondisk_log_size":9721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102825984,"num_objects":1569,"num_object_clones":0,"num_object_copies":4707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2051,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9721,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4077,"num_bytes_recovered":267190272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":102},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":584}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":360,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361027+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:48:10.361027+0000","last_peered":"2024-09-13T14:48:10.361027+0000","last_clean":"2024-09-13T14:48:10.361027+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:48:10.361027+0000","last_undegraded":"2024-09-13T14:48:10.361027+0000","last_fullsized":"2024-09-13T14:48:10.361027+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350235+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:48:10.350235+0000","last_peered":"2024-09-13T14:48:10.350235+0000","last_clean":"2024-09-13T14:48:10.350235+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:48:10.350235+0000","last_undegraded":"2024-09-13T14:48:10.350235+0000","last_fullsized":"2024-09-13T14:48:10.350235+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"367'9841","reported_seq":19405,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:27.178636+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:48:27.178636+0000","last_peered":"2024-09-13T14:48:27.178636+0000","last_clean":"2024-09-13T14:48:27.178636+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:48:27.178636+0000","last_undegraded":"2024-09-13T14:48:27.178636+0000","last_fullsized":"2024-09-13T14:48:27.178636+0000","mapping_epoch":326,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9841,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":378,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361259+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:48:10.361259+0000","last_peered":"2024-09-13T14:48:10.361259+0000","last_clean":"2024-09-13T14:48:10.361259+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:48:10.361259+0000","last_undegraded":"2024-09-13T14:48:10.361259+0000","last_fullsized":"2024-09-13T14:48:10.361259+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346393+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:48:10.346393+0000","last_peered":"2024-09-13T14:48:10.346393+0000","last_clean":"2024-09-13T14:48:10.346393+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:48:10.346393+0000","last_undegraded":"2024-09-13T14:48:10.346393+0000","last_fullsized":"2024-09-13T14:48:10.346393+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"367'10192","reported_seq":10335,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:28.643871+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:48:28.643871+0000","last_peered":"2024-09-13T14:48:28.643871+0000","last_clean":"2024-09-13T14:48:28.643871+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:48:28.643871+0000","last_undegraded":"2024-09-13T14:48:28.643871+0000","last_fullsized":"2024-09-13T14:48:28.643871+0000","mapping_epoch":303,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8592,"log_dups_size":0,"ondisk_log_size":8592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105250816,"num_objects":1606,"num_object_clones":0,"num_object_copies":4818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10192,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":332,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361062+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:48:10.361062+0000","last_peered":"2024-09-13T14:48:10.361062+0000","last_clean":"2024-09-13T14:48:10.361062+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:48:10.361062+0000","last_undegraded":"2024-09-13T14:48:10.361062+0000","last_fullsized":"2024-09-13T14:48:10.361062+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":347,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345480+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:48:10.345480+0000","last_peered":"2024-09-13T14:48:10.345480+0000","last_clean":"2024-09-13T14:48:10.345480+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:48:10.345480+0000","last_undegraded":"2024-09-13T14:48:10.345480+0000","last_fullsized":"2024-09-13T14:48:10.345480+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"367'9882","reported_seq":10025,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:24.915751+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:48:24.915751+0000","last_peered":"2024-09-13T14:48:24.915751+0000","last_clean":"2024-09-13T14:48:24.915751+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:48:24.915751+0000","last_undegraded":"2024-09-13T14:48:24.915751+0000","last_fullsized":"2024-09-13T14:48:24.915751+0000","mapping_epoch":303,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109051904,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9882,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":295,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350890+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:48:10.350890+0000","last_peered":"2024-09-13T14:48:10.350890+0000","last_clean":"2024-09-13T14:48:10.350890+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:48:10.350890+0000","last_undegraded":"2024-09-13T14:48:10.350890+0000","last_fullsized":"2024-09-13T14:48:10.350890+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":630,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341526+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:48:20.341526+0000","last_peered":"2024-09-13T14:48:20.341526+0000","last_clean":"2024-09-13T14:48:20.341526+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:48:20.341526+0000","last_undegraded":"2024-09-13T14:48:20.341526+0000","last_fullsized":"2024-09-13T14:48:20.341526+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"367'9818","reported_seq":9958,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:25.644258+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:48:25.644258+0000","last_peered":"2024-09-13T14:48:25.644258+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:48:25.644258+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'2800","ondisk_log_start":"307'2800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107741184,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9818,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-3989}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":478,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360386+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:48:10.360386+0000","last_peered":"2024-09-13T14:48:10.360386+0000","last_clean":"2024-09-13T14:48:10.360386+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:48:10.360386+0000","last_undegraded":"2024-09-13T14:48:10.360386+0000","last_fullsized":"2024-09-13T14:48:10.360386+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":628,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.342373+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:48:20.342373+0000","last_peered":"2024-09-13T14:48:20.342373+0000","last_clean":"2024-09-13T14:48:20.342373+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:48:20.342373+0000","last_undegraded":"2024-09-13T14:48:20.342373+0000","last_fullsized":"2024-09-13T14:48:20.342373+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346543+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:48:10.346543+0000","last_peered":"2024-09-13T14:48:10.346543+0000","last_clean":"2024-09-13T14:48:10.346543+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:48:10.346543+0000","last_undegraded":"2024-09-13T14:48:10.346543+0000","last_fullsized":"2024-09-13T14:48:10.346543+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"367'9500","reported_seq":9643,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:24.704840+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:48:24.704840+0000","last_peered":"2024-09-13T14:48:24.704840+0000","last_clean":"2024-09-13T14:48:24.704840+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:48:24.704840+0000","last_undegraded":"2024-09-13T14:48:24.704840+0000","last_fullsized":"2024-09-13T14:48:24.704840+0000","mapping_epoch":303,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104464384,"num_objects":1594,"num_object_clones":0,"num_object_copies":4782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9500,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":182,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350332+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:48:10.350332+0000","last_peered":"2024-09-13T14:48:10.350332+0000","last_clean":"2024-09-13T14:48:10.350332+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:48:10.350332+0000","last_undegraded":"2024-09-13T14:48:10.350332+0000","last_fullsized":"2024-09-13T14:48:10.350332+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":368,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345357+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:48:10.345357+0000","last_peered":"2024-09-13T14:48:10.345357+0000","last_clean":"2024-09-13T14:48:10.345357+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:48:10.345357+0000","last_undegraded":"2024-09-13T14:48:10.345357+0000","last_fullsized":"2024-09-13T14:48:10.345357+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345870+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:48:10.345870+0000","last_peered":"2024-09-13T14:48:10.345870+0000","last_clean":"2024-09-13T14:48:10.345870+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:48:10.345870+0000","last_undegraded":"2024-09-13T14:48:10.345870+0000","last_fullsized":"2024-09-13T14:48:10.345870+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"367'9780","reported_seq":21730,"reported_epoch":367,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:27.656445+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:48:27.656445+0000","last_peered":"2024-09-13T14:48:27.656445+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:48:27.656445+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7380,"log_dups_size":0,"ondisk_log_size":7380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":577,"num_objects_missing":577,"num_objects_degraded":952,"num_objects_misplaced":1529,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9780,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4074,"num_bytes_recovered":266993664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":577},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-272}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346142+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:48:10.346142+0000","last_peered":"2024-09-13T14:48:10.346142+0000","last_clean":"2024-09-13T14:48:10.346142+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:48:10.346142+0000","last_undegraded":"2024-09-13T14:48:10.346142+0000","last_fullsized":"2024-09-13T14:48:10.346142+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345564+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:48:10.345564+0000","last_peered":"2024-09-13T14:48:10.345564+0000","last_clean":"2024-09-13T14:48:10.345564+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:48:10.345564+0000","last_undegraded":"2024-09-13T14:48:10.345564+0000","last_fullsized":"2024-09-13T14:48:10.345564+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"367'9502","reported_seq":9645,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:24.951437+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:48:24.951437+0000","last_peered":"2024-09-13T14:48:24.951437+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:48:24.951437+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'2000","ondisk_log_start":"307'2000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7502,"log_dups_size":0,"ondisk_log_size":7502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108527616,"num_objects":1656,"num_object_clones":0,"num_object_copies":4968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9502,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-3810}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":175,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350523+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:48:10.350523+0000","last_peered":"2024-09-13T14:48:10.350523+0000","last_clean":"2024-09-13T14:48:10.350523+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:48:10.350523+0000","last_undegraded":"2024-09-13T14:48:10.350523+0000","last_fullsized":"2024-09-13T14:48:10.350523+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":621,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.359118+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:48:20.359118+0000","last_peered":"2024-09-13T14:48:20.359118+0000","last_clean":"2024-09-13T14:48:20.359118+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:48:20.359118+0000","last_undegraded":"2024-09-13T14:48:20.359118+0000","last_fullsized":"2024-09-13T14:48:20.359118+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":583,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.346727+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:48:20.346727+0000","last_peered":"2024-09-13T14:48:20.346727+0000","last_clean":"2024-09-13T14:48:20.346727+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:48:20.346727+0000","last_undegraded":"2024-09-13T14:48:20.346727+0000","last_fullsized":"2024-09-13T14:48:20.346727+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"367'9645","reported_seq":9724,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:27.014414+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:48:27.014414+0000","last_peered":"2024-09-13T14:48:27.014414+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:48:27.014414+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9645,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-3921}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":469,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360520+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:48:10.360520+0000","last_peered":"2024-09-13T14:48:10.360520+0000","last_clean":"2024-09-13T14:48:10.360520+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:48:10.360520+0000","last_undegraded":"2024-09-13T14:48:10.360520+0000","last_fullsized":"2024-09-13T14:48:10.360520+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":891,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350620+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:48:10.350620+0000","last_peered":"2024-09-13T14:48:10.350620+0000","last_clean":"2024-09-13T14:48:10.350620+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:48:10.350620+0000","last_undegraded":"2024-09-13T14:48:10.350620+0000","last_fullsized":"2024-09-13T14:48:10.350620+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":574,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.342804+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:48:20.342804+0000","last_peered":"2024-09-13T14:48:20.342804+0000","last_clean":"2024-09-13T14:48:20.342804+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:48:20.342804+0000","last_undegraded":"2024-09-13T14:48:20.342804+0000","last_fullsized":"2024-09-13T14:48:20.342804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":333,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346836+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:48:10.346836+0000","last_peered":"2024-09-13T14:48:10.346836+0000","last_clean":"2024-09-13T14:48:10.346836+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:48:10.346836+0000","last_undegraded":"2024-09-13T14:48:10.346836+0000","last_fullsized":"2024-09-13T14:48:10.346836+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"367'9904","reported_seq":10045,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:24.911578+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:48:24.911578+0000","last_peered":"2024-09-13T14:48:24.911578+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:48:24.911578+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7504,"log_dups_size":0,"ondisk_log_size":7504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107741184,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9904,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-4036}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":428,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.350138+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:48:20.350138+0000","last_peered":"2024-09-13T14:48:20.350138+0000","last_clean":"2024-09-13T14:48:20.350138+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:48:20.350138+0000","last_undegraded":"2024-09-13T14:48:20.350138+0000","last_fullsized":"2024-09-13T14:48:20.350138+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":757,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360475+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:48:10.360475+0000","last_peered":"2024-09-13T14:48:10.360475+0000","last_clean":"2024-09-13T14:48:10.360475+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:48:10.360475+0000","last_undegraded":"2024-09-13T14:48:10.360475+0000","last_fullsized":"2024-09-13T14:48:10.360475+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":623,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341567+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:48:20.341567+0000","last_peered":"2024-09-13T14:48:20.341567+0000","last_clean":"2024-09-13T14:48:20.341567+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:48:20.341567+0000","last_undegraded":"2024-09-13T14:48:20.341567+0000","last_fullsized":"2024-09-13T14:48:20.341567+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":628,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341663+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:48:20.341663+0000","last_peered":"2024-09-13T14:48:20.341663+0000","last_clean":"2024-09-13T14:48:20.341663+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:48:20.341663+0000","last_undegraded":"2024-09-13T14:48:20.341663+0000","last_fullsized":"2024-09-13T14:48:20.341663+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"367'9933","reported_seq":19616,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:25.644029+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:48:25.644029+0000","last_peered":"2024-09-13T14:48:25.644029+0000","last_clean":"2024-09-13T14:48:25.644029+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:48:25.644029+0000","last_undegraded":"2024-09-13T14:48:25.644029+0000","last_fullsized":"2024-09-13T14:48:25.644029+0000","mapping_epoch":330,"log_start":"307'2900","ondisk_log_start":"307'2900","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113442816,"num_objects":1731,"num_object_clones":0,"num_object_copies":5193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9933,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":498,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360641+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:48:10.360641+0000","last_peered":"2024-09-13T14:48:10.360641+0000","last_clean":"2024-09-13T14:48:10.360641+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:48:10.360641+0000","last_undegraded":"2024-09-13T14:48:10.360641+0000","last_fullsized":"2024-09-13T14:48:10.360641+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":592,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.342010+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:48:20.342010+0000","last_peered":"2024-09-13T14:48:20.342010+0000","last_clean":"2024-09-13T14:48:20.342010+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:48:20.342010+0000","last_undegraded":"2024-09-13T14:48:20.342010+0000","last_fullsized":"2024-09-13T14:48:20.342010+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":560,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.347700+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:48:20.347700+0000","last_peered":"2024-09-13T14:48:20.347700+0000","last_clean":"2024-09-13T14:48:20.347700+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:48:20.347700+0000","last_undegraded":"2024-09-13T14:48:20.347700+0000","last_fullsized":"2024-09-13T14:48:20.347700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":116,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350380+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:48:10.350380+0000","last_peered":"2024-09-13T14:48:10.350380+0000","last_clean":"2024-09-13T14:48:10.350380+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:48:10.350380+0000","last_undegraded":"2024-09-13T14:48:10.350380+0000","last_fullsized":"2024-09-13T14:48:10.350380+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"367'9612","reported_seq":9754,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:25.677561+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:48:25.677561+0000","last_peered":"2024-09-13T14:48:25.677561+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:48:25.677561+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'2600","ondisk_log_start":"307'2600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106037248,"num_objects":1618,"num_object_clones":0,"num_object_copies":4854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9612,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-3891}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":175,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350368+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:48:10.350368+0000","last_peered":"2024-09-13T14:48:10.350368+0000","last_clean":"2024-09-13T14:48:10.350368+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:48:10.350368+0000","last_undegraded":"2024-09-13T14:48:10.350368+0000","last_fullsized":"2024-09-13T14:48:10.350368+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":982,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350581+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:48:10.350581+0000","last_peered":"2024-09-13T14:48:10.350581+0000","last_clean":"2024-09-13T14:48:10.350581+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:48:10.350581+0000","last_undegraded":"2024-09-13T14:48:10.350581+0000","last_fullsized":"2024-09-13T14:48:10.350581+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346142+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:48:10.346142+0000","last_peered":"2024-09-13T14:48:10.346142+0000","last_clean":"2024-09-13T14:48:10.346142+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:48:10.346142+0000","last_undegraded":"2024-09-13T14:48:10.346142+0000","last_fullsized":"2024-09-13T14:48:10.346142+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":581,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.345765+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:48:20.345765+0000","last_peered":"2024-09-13T14:48:20.345765+0000","last_clean":"2024-09-13T14:48:20.345765+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:48:20.345765+0000","last_undegraded":"2024-09-13T14:48:20.345765+0000","last_fullsized":"2024-09-13T14:48:20.345765+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"367'9712","reported_seq":19134,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:28.658331+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:48:28.658331+0000","last_peered":"2024-09-13T14:48:28.658331+0000","last_clean":"2024-09-13T14:48:28.658331+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:48:28.658331+0000","last_undegraded":"2024-09-13T14:48:28.658331+0000","last_fullsized":"2024-09-13T14:48:28.658331+0000","mapping_epoch":337,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108003328,"num_objects":1648,"num_object_clones":0,"num_object_copies":4944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9712,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":366,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346266+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:48:10.346266+0000","last_peered":"2024-09-13T14:48:10.346266+0000","last_clean":"2024-09-13T14:48:10.346266+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:48:10.346266+0000","last_undegraded":"2024-09-13T14:48:10.346266+0000","last_fullsized":"2024-09-13T14:48:10.346266+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":312,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350762+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:48:10.350762+0000","last_peered":"2024-09-13T14:48:10.350762+0000","last_clean":"2024-09-13T14:48:10.350762+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:48:10.350762+0000","last_undegraded":"2024-09-13T14:48:10.350762+0000","last_fullsized":"2024-09-13T14:48:10.350762+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1160,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.345982+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:48:20.345982+0000","last_peered":"2024-09-13T14:48:20.345982+0000","last_clean":"2024-09-13T14:48:20.345982+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:48:20.345982+0000","last_undegraded":"2024-09-13T14:48:20.345982+0000","last_fullsized":"2024-09-13T14:48:20.345982+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":562,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.348165+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:48:20.348165+0000","last_peered":"2024-09-13T14:48:20.348165+0000","last_clean":"2024-09-13T14:48:20.348165+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:48:20.348165+0000","last_undegraded":"2024-09-13T14:48:20.348165+0000","last_fullsized":"2024-09-13T14:48:20.348165+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"367'9573","reported_seq":9715,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:25.691897+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:48:25.691897+0000","last_peered":"2024-09-13T14:48:25.691897+0000","last_clean":"2024-09-13T14:48:25.691897+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:48:25.691897+0000","last_undegraded":"2024-09-13T14:48:25.691897+0000","last_fullsized":"2024-09-13T14:48:25.691897+0000","mapping_epoch":303,"log_start":"307'2500","ondisk_log_start":"307'2500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106889216,"num_objects":1631,"num_object_clones":0,"num_object_copies":4893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9573,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":366,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345961+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:48:10.345961+0000","last_peered":"2024-09-13T14:48:10.345961+0000","last_clean":"2024-09-13T14:48:10.345961+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:48:10.345961+0000","last_undegraded":"2024-09-13T14:48:10.345961+0000","last_fullsized":"2024-09-13T14:48:10.345961+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":571,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.345867+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:48:20.345867+0000","last_peered":"2024-09-13T14:48:20.345867+0000","last_clean":"2024-09-13T14:48:20.345867+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:48:20.345867+0000","last_undegraded":"2024-09-13T14:48:20.345867+0000","last_fullsized":"2024-09-13T14:48:20.345867+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":761,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.361263+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:48:10.361263+0000","last_peered":"2024-09-13T14:48:10.361263+0000","last_clean":"2024-09-13T14:48:10.361263+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:48:10.361263+0000","last_undegraded":"2024-09-13T14:48:10.361263+0000","last_fullsized":"2024-09-13T14:48:10.361263+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":606,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341427+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:48:20.341427+0000","last_peered":"2024-09-13T14:48:20.341427+0000","last_clean":"2024-09-13T14:48:20.341427+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:48:20.341427+0000","last_undegraded":"2024-09-13T14:48:20.341427+0000","last_fullsized":"2024-09-13T14:48:20.341427+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"367'10088","reported_seq":10230,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:28.782338+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:48:28.782338+0000","last_peered":"2024-09-13T14:48:28.782338+0000","last_clean":"2024-09-13T14:48:28.782338+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:48:28.782338+0000","last_undegraded":"2024-09-13T14:48:28.782338+0000","last_fullsized":"2024-09-13T14:48:28.782338+0000","mapping_epoch":303,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104988672,"num_objects":1602,"num_object_clones":0,"num_object_copies":4806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10088,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345493+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:48:10.345493+0000","last_peered":"2024-09-13T14:48:10.345493+0000","last_clean":"2024-09-13T14:48:10.345493+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:48:10.345493+0000","last_undegraded":"2024-09-13T14:48:10.345493+0000","last_fullsized":"2024-09-13T14:48:10.345493+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":607,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.343472+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:48:20.343472+0000","last_peered":"2024-09-13T14:48:20.343472+0000","last_clean":"2024-09-13T14:48:20.343472+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:48:20.343472+0000","last_undegraded":"2024-09-13T14:48:20.343472+0000","last_fullsized":"2024-09-13T14:48:20.343472+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":947,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.347831+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:48:20.347831+0000","last_peered":"2024-09-13T14:48:20.347831+0000","last_clean":"2024-09-13T14:48:20.347831+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:48:20.347831+0000","last_undegraded":"2024-09-13T14:48:20.347831+0000","last_fullsized":"2024-09-13T14:48:20.347831+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":605,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.358835+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:48:20.358835+0000","last_peered":"2024-09-13T14:48:20.358835+0000","last_clean":"2024-09-13T14:48:20.358835+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:48:20.358835+0000","last_undegraded":"2024-09-13T14:48:20.358835+0000","last_fullsized":"2024-09-13T14:48:20.358835+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"367'9948","reported_seq":21326,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:27.731970+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:48:27.731970+0000","last_peered":"2024-09-13T14:48:27.731970+0000","last_clean":"2024-09-13T14:48:27.731970+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:48:27.731970+0000","last_undegraded":"2024-09-13T14:48:27.731970+0000","last_fullsized":"2024-09-13T14:48:27.731970+0000","mapping_epoch":332,"log_start":"306'800","ondisk_log_start":"306'800","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102563864,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9948,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":458,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341931+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:48:20.341931+0000","last_peered":"2024-09-13T14:48:20.341931+0000","last_clean":"2024-09-13T14:48:20.341931+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:48:20.341931+0000","last_undegraded":"2024-09-13T14:48:20.341931+0000","last_fullsized":"2024-09-13T14:48:20.341931+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":642,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341366+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:48:20.341366+0000","last_peered":"2024-09-13T14:48:20.341366+0000","last_clean":"2024-09-13T14:48:20.341366+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:48:20.341366+0000","last_undegraded":"2024-09-13T14:48:20.341366+0000","last_fullsized":"2024-09-13T14:48:20.341366+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":785,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346414+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:48:10.346414+0000","last_peered":"2024-09-13T14:48:10.346414+0000","last_clean":"2024-09-13T14:48:10.346414+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:48:10.346414+0000","last_undegraded":"2024-09-13T14:48:10.346414+0000","last_fullsized":"2024-09-13T14:48:10.346414+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":80,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346269+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:48:10.346269+0000","last_peered":"2024-09-13T14:48:10.346269+0000","last_clean":"2024-09-13T14:48:10.346269+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:48:10.346269+0000","last_undegraded":"2024-09-13T14:48:10.346269+0000","last_fullsized":"2024-09-13T14:48:10.346269+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346221+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:48:10.346221+0000","last_peered":"2024-09-13T14:48:10.346221+0000","last_clean":"2024-09-13T14:48:10.346221+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:48:10.346221+0000","last_undegraded":"2024-09-13T14:48:10.346221+0000","last_fullsized":"2024-09-13T14:48:10.346221+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":76,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346397+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:48:10.346397+0000","last_peered":"2024-09-13T14:48:10.346397+0000","last_clean":"2024-09-13T14:48:10.346397+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:48:10.346397+0000","last_undegraded":"2024-09-13T14:48:10.346397+0000","last_fullsized":"2024-09-13T14:48:10.346397+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":558,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.348018+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:48:20.348018+0000","last_peered":"2024-09-13T14:48:20.348018+0000","last_clean":"2024-09-13T14:48:20.348018+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:48:20.348018+0000","last_undegraded":"2024-09-13T14:48:20.348018+0000","last_fullsized":"2024-09-13T14:48:20.348018+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":366,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.360971+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:48:10.360971+0000","last_peered":"2024-09-13T14:48:10.360971+0000","last_clean":"2024-09-13T14:48:10.360971+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:48:10.360971+0000","last_undegraded":"2024-09-13T14:48:10.360971+0000","last_fullsized":"2024-09-13T14:48:10.360971+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":332,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350407+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:48:10.350407+0000","last_peered":"2024-09-13T14:48:10.350407+0000","last_clean":"2024-09-13T14:48:10.350407+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:48:10.350407+0000","last_undegraded":"2024-09-13T14:48:10.350407+0000","last_fullsized":"2024-09-13T14:48:10.350407+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":563,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341707+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:48:20.341707+0000","last_peered":"2024-09-13T14:48:20.341707+0000","last_clean":"2024-09-13T14:48:20.341707+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:48:20.341707+0000","last_undegraded":"2024-09-13T14:48:20.341707+0000","last_fullsized":"2024-09-13T14:48:20.341707+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":615,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341276+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:48:20.341276+0000","last_peered":"2024-09-13T14:48:20.341276+0000","last_clean":"2024-09-13T14:48:20.341276+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:48:20.341276+0000","last_undegraded":"2024-09-13T14:48:20.341276+0000","last_fullsized":"2024-09-13T14:48:20.341276+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":619,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.358728+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:48:20.358728+0000","last_peered":"2024-09-13T14:48:20.358728+0000","last_clean":"2024-09-13T14:48:20.358728+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:48:20.358728+0000","last_undegraded":"2024-09-13T14:48:20.358728+0000","last_fullsized":"2024-09-13T14:48:20.358728+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":364,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346339+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:48:10.346339+0000","last_peered":"2024-09-13T14:48:10.346339+0000","last_clean":"2024-09-13T14:48:10.346339+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:48:10.346339+0000","last_undegraded":"2024-09-13T14:48:10.346339+0000","last_fullsized":"2024-09-13T14:48:10.346339+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":637,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.359007+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:48:20.359007+0000","last_peered":"2024-09-13T14:48:20.359007+0000","last_clean":"2024-09-13T14:48:20.359007+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:48:20.359007+0000","last_undegraded":"2024-09-13T14:48:20.359007+0000","last_fullsized":"2024-09-13T14:48:20.359007+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":607,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341572+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:48:20.341572+0000","last_peered":"2024-09-13T14:48:20.341572+0000","last_clean":"2024-09-13T14:48:20.341572+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:48:20.341572+0000","last_undegraded":"2024-09-13T14:48:20.341572+0000","last_fullsized":"2024-09-13T14:48:20.341572+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":583,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.348004+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:48:20.348004+0000","last_peered":"2024-09-13T14:48:20.348004+0000","last_clean":"2024-09-13T14:48:20.348004+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:48:20.348004+0000","last_undegraded":"2024-09-13T14:48:20.348004+0000","last_fullsized":"2024-09-13T14:48:20.348004+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":632,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341714+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:48:20.341714+0000","last_peered":"2024-09-13T14:48:20.341714+0000","last_clean":"2024-09-13T14:48:20.341714+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:48:20.341714+0000","last_undegraded":"2024-09-13T14:48:20.341714+0000","last_fullsized":"2024-09-13T14:48:20.341714+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":65,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.355417+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:48:10.355417+0000","last_peered":"2024-09-13T14:48:10.355417+0000","last_clean":"2024-09-13T14:48:10.355417+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:48:10.355417+0000","last_undegraded":"2024-09-13T14:48:10.355417+0000","last_fullsized":"2024-09-13T14:48:10.355417+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":529,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.346126+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:48:20.346126+0000","last_peered":"2024-09-13T14:48:20.346126+0000","last_clean":"2024-09-13T14:48:20.346126+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:48:20.346126+0000","last_undegraded":"2024-09-13T14:48:20.346126+0000","last_fullsized":"2024-09-13T14:48:20.346126+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":610,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341053+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:48:20.341053+0000","last_peered":"2024-09-13T14:48:20.341053+0000","last_clean":"2024-09-13T14:48:20.341053+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:48:20.341053+0000","last_undegraded":"2024-09-13T14:48:20.341053+0000","last_fullsized":"2024-09-13T14:48:20.341053+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.346062+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:48:10.346062+0000","last_peered":"2024-09-13T14:48:10.346062+0000","last_clean":"2024-09-13T14:48:10.346062+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:48:10.346062+0000","last_undegraded":"2024-09-13T14:48:10.346062+0000","last_fullsized":"2024-09-13T14:48:10.346062+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":332,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350687+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:48:10.350687+0000","last_peered":"2024-09-13T14:48:10.350687+0000","last_clean":"2024-09-13T14:48:10.350687+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:48:10.350687+0000","last_undegraded":"2024-09-13T14:48:10.350687+0000","last_fullsized":"2024-09-13T14:48:10.350687+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":65,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.355339+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:48:10.355339+0000","last_peered":"2024-09-13T14:48:10.355339+0000","last_clean":"2024-09-13T14:48:10.355339+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:48:10.355339+0000","last_undegraded":"2024-09-13T14:48:10.355339+0000","last_fullsized":"2024-09-13T14:48:10.355339+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":298,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.350855+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:48:10.350855+0000","last_peered":"2024-09-13T14:48:10.350855+0000","last_clean":"2024-09-13T14:48:10.350855+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:48:10.350855+0000","last_undegraded":"2024-09-13T14:48:10.350855+0000","last_fullsized":"2024-09-13T14:48:10.350855+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":629,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.342108+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:48:20.342108+0000","last_peered":"2024-09-13T14:48:20.342108+0000","last_clean":"2024-09-13T14:48:20.342108+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:48:20.342108+0000","last_undegraded":"2024-09-13T14:48:20.342108+0000","last_fullsized":"2024-09-13T14:48:20.342108+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":616,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.343078+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:48:20.343078+0000","last_peered":"2024-09-13T14:48:20.343078+0000","last_clean":"2024-09-13T14:48:20.343078+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:48:20.343078+0000","last_undegraded":"2024-09-13T14:48:20.343078+0000","last_fullsized":"2024-09-13T14:48:20.343078+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-09-13T14:48:10.345264+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:48:10.345264+0000","last_peered":"2024-09-13T14:48:10.345264+0000","last_clean":"2024-09-13T14:48:10.345264+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:48:10.345264+0000","last_undegraded":"2024-09-13T14:48:10.345264+0000","last_fullsized":"2024-09-13T14:48:10.345264+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":620,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.341164+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:48:20.341164+0000","last_peered":"2024-09-13T14:48:20.341164+0000","last_clean":"2024-09-13T14:48:20.341164+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:48:20.341164+0000","last_undegraded":"2024-09-13T14:48:20.341164+0000","last_fullsized":"2024-09-13T14:48:20.341164+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1705639960,"num_objects":26027,"num_object_clones":0,"num_object_copies":78081,"num_objects_missing_on_primary":577,"num_objects_missing":577,"num_objects_degraded":30787,"num_objects_misplaced":1529,"num_objects_unfound":0,"num_objects_dirty":26027,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156651,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27948,"num_bytes_recovered":1771175936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5377437696,"data_stored":5377425480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125451,"ondisk_log_size":125451,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397948,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1416684,"kb_used_data":524836,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":92328468,"statfs":{"total":95995035648,"available":94544351232,"internally_reserved":0,"allocated":537432064,"data_stored":535459256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594176,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1461528,"kb_used_data":875792,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":92283624,"statfs":{"total":95995035648,"available":94498430976,"internally_reserved":0,"allocated":896811008,"data_stored":894857954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665630,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484632,"kb_used_data":152080,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":93260520,"statfs":{"total":95995035648,"available":95498772480,"internally_reserved":0,"allocated":155729920,"data_stored":153773197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986632,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365988,"kb_used_data":457820,"kb_used_omap":1,"kb_used_meta":908158,"kb_avail":92379164,"statfs":{"total":95995035648,"available":94596263936,"internally_reserved":0,"allocated":468807680,"data_stored":466842970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":929954136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,0,2,2,2,6,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697191,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1754004,"kb_used_data":1004172,"kb_used_omap":4,"kb_used_meta":749819,"kb_avail":91991148,"statfs":{"total":95995035648,"available":94198935552,"internally_reserved":0,"allocated":1028272128,"data_stored":1026319558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":767814731},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860718,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564316,"kb_used_data":929300,"kb_used_omap":8,"kb_used_meta":634999,"kb_avail":92180836,"statfs":{"total":95995035648,"available":94393176064,"internally_reserved":0,"allocated":951603200,"data_stored":949642976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":650239820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020389,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1066640,"kb_used_data":753352,"kb_used_omap":9,"kb_used_meta":313270,"kb_avail":92678512,"statfs":{"total":95995035648,"available":94902796288,"internally_reserved":0,"allocated":771432448,"data_stored":769483351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":320789317},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187780,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1261456,"kb_used_data":621064,"kb_used_omap":7,"kb_used_meta":640376,"kb_avail":92483696,"statfs":{"total":95995035648,"available":94703304704,"internally_reserved":0,"allocated":635969536,"data_stored":634020705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":655745355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":625541120,"data_stored":625541120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":760807424,"data_stored":760807424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":939921408,"data_stored":939917336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1019084800,"data_stored":1019084800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":462225408,"data_stored":462225408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":150343680,"data_stored":150339608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":891748352,"data_stored":891748352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":527765504,"data_stored":527761432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:48:31.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:48:31.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.039 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:32.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:32.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:32.189 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:32 smithi189 ceph-mon[90992]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 30787/75909 objects degraded (40.558%); 1529/75909 objects misplaced (2.014%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T14:48:32.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:32.323 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:32.336 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:32 smithi017 ceph-mon[103001]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 30787/75909 objects degraded (40.558%); 1529/75909 objects misplaced (2.014%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T14:48:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:32 smithi017 ceph-mon[107556]: pgmap v1198: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 30787/75909 objects degraded (40.558%); 1529/75909 objects misplaced (2.014%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:158299 8.f e367/303 8:feb6c2ba:::benchmark_data_smithi017_253885_object66958:head v 367'9693, mlcod=367'9691)", 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:32.331129+0000", 2024-09-13T14:48:32.340 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.003823318, 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003850479, 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331129+0000", 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331129+0000", 2024-09-13T14:48:32.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331131+0000", 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7e-06 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331142+0000", 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0665e-05 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:48:32.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331161+0000", 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9766999999999999e-05 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331172+0000", 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0671e-05 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331272+0000", 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9864000000000005e-05 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:48:32.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:32.331295+0000", 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2864999999999999e-05 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 1 2024-09-13T14:48:32.344 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:48:32.420 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:32.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:32.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:32.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:32.596 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:32.670 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:32.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:32.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:32.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:33.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:33.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:33.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:33.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:33.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:48:33.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:33.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:33.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:33.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:33.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:33.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:33.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:33.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:34.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.347 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:48:34.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:34.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:34.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:34.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[103001]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 30527/74577 objects degraded (40.934%); 1443/74577 objects misplaced (1.935%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 30527/74577 objects degraded (40.934%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[107556]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 30527/74577 objects degraded (40.934%); 1443/74577 objects misplaced (1.935%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 30527/74577 objects degraded (40.934%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:34.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:34.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:34.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:34 smithi189 ceph-mon[90992]: pgmap v1199: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 30527/74577 objects degraded (40.934%); 1443/74577 objects misplaced (1.935%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:48:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 30527/74577 objects degraded (40.934%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:34.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:34.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:34.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:35.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:35.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:35.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:35.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:35.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:35.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:35.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:35.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[103001]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:48:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:35 smithi017 ceph-mon[107556]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:48:35.736 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:35 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3867659336' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:35 smithi189 ceph-mon[90992]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:48:35.759 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:35.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.234 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:36.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:36.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:36.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:36.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:36.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.511 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:36.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.639 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:36 smithi017 ceph-mon[103001]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 30527/72693 objects degraded (41.994%); 1443/72693 objects misplaced (1.985%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:36 smithi017 ceph-mon[107556]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 30527/72693 objects degraded (41.994%); 1443/72693 objects misplaced (1.985%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:36.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:36 smithi189 ceph-mon[90992]: pgmap v1201: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 30527/72693 objects degraded (41.994%); 1443/72693 objects misplaced (1.985%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:48:36.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:36.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:36.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:36.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:37.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:37.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:37.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:37.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:37.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.275 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:48:37.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:37.342 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:48:37.342 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:37.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:48:37.364 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:37.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.384 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:48:37.384 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:37.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.401 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:48:37.401 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:37.420 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:48:37.420 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:37.436 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:48:37.436 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:37.451 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:48:37.452 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:37.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:37.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:37.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2922069847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:37] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-13T14:48:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2922069847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:37.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2922069847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:37.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:37.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:37.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:38.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.280 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:38.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:38.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:38 smithi017 ceph-mon[103001]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 30335/71178 objects degraded (42.619%); 1443/71178 objects misplaced (2.027%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:48:38.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:38 smithi017 ceph-mon[107556]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 30335/71178 objects degraded (42.619%); 1443/71178 objects misplaced (2.027%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:48:38.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.659 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:38 smithi189 ceph-mon[90992]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 30335/71178 objects degraded (42.619%); 1443/71178 objects misplaced (2.027%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:48:38.761 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:38.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:38.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:48:38.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.914 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.961 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:38.961 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30335/71178 objects degraded (42.619%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1716,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":368,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":23726,"data_bytes":1557695158,"bytes_used":10281295872,"bytes_avail":757678989312,"bytes_total":767960285184,"degraded_objects":30335,"degraded_total":71178,"degraded_ratio":0.42618505718058958,"misplaced_objects":1443,"misplaced_total":71178,"misplaced_ratio":0.020273118098288797,"recovering_objects_per_sec":35,"recovering_bytes_per_sec":2319290,"recovering_keys_per_sec":0,"num_objects_recovered":354,"num_bytes_recovered":23199744,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":291},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (9m)\n [==========================..] (remaining: 42s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:48:39.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:39.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:39.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:39.446 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:39.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:39.512 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:39.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:39.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:39.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 30335/71178 objects degraded (42.619%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:39.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/935683850' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:39.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 30335/71178 objects degraded (42.619%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:39.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/935683850' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:39.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:39.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 30335/71178 objects degraded (42.619%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/935683850' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:48:39.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.842 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:39.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:39.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:40.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.073 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:48:40.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:48:40.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.200 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:40.222 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:40.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:40.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:40.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:40.477 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:40.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:40 smithi189 ceph-mon[90992]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 30245/69114 objects degraded (43.761%); 1350/69114 objects misplaced (1.953%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:40.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:40.603 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:40 smithi017 ceph-mon[107556]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 30245/69114 objects degraded (43.761%); 1350/69114 objects misplaced (1.953%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:40 smithi017 ceph-mon[103001]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 30245/69114 objects degraded (43.761%); 1350/69114 objects misplaced (1.953%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:40.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:40.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:40.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:40.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:40.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:40.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:41.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:41.030 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:41.081 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:41.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:41.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:41.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:41.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:41.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:41.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:41.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:41.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.121 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:42.145 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:42.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:42.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.373 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:42.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:42.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:42.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[103001]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 308 op/s; 30245/66660 objects degraded (45.372%); 1350/66660 objects misplaced (2.025%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:42.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:42.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[107556]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 308 op/s; 30245/66660 objects degraded (45.372%); 1350/66660 objects misplaced (2.025%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:42.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:42 smithi189 ceph-mon[90992]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 308 op/s; 30245/66660 objects degraded (45.372%); 1350/66660 objects misplaced (2.025%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:48:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:42.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:42.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:42.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:42.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:43.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:43.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:43.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:43.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:43.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:43.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:43.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:43.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:43.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.649 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:43.649 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:43.671 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:43.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:43.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:44.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:44.148 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:44.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:44.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:44.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:44.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:48:44.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:44.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:44.528 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:44.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:44.650 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:44.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:44.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:44 smithi017 ceph-mon[103001]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 29914/65076 objects degraded (45.968%); 1254/65076 objects misplaced (1.927%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29914/65076 objects degraded (45.968%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:44 smithi017 ceph-mon[107556]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 29914/65076 objects degraded (45.968%); 1254/65076 objects misplaced (1.927%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29914/65076 objects degraded (45.968%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:44 smithi189 ceph-mon[90992]: pgmap v1205: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 316 op/s; 29914/65076 objects degraded (45.968%); 1254/65076 objects misplaced (1.927%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29914/65076 objects degraded (45.968%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:44.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:44.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:44.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:45.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:45.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:45.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:45.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:45.423 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:45.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:45.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:45.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:45.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:45.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:45.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:46.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:46.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:46.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.239 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:46.240 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:48:46.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:46.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:46.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:46.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:46.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:46.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:46 smithi017 ceph-mon[103001]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 29914/62997 objects degraded (47.485%); 1254/62997 objects misplaced (1.991%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:46 smithi017 ceph-mon[107556]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 29914/62997 objects degraded (47.485%); 1254/62997 objects misplaced (1.991%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:46.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.709 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:46.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:46 smithi189 ceph-mon[90992]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 326 op/s; 29914/62997 objects degraded (47.485%); 1254/62997 objects misplaced (1.991%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:48:46.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:46.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:46.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.108 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:47.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:47.337 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.364 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:47.384 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:47] "GET /metrics HTTP/1.1" 200 36700 "" "Prometheus/2.51.0" 2024-09-13T14:48:47.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:47.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:47.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:47.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:47 smithi017 ceph-mon[103001]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:47.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:47 smithi017 ceph-mon[107556]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:47.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:47.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:47 smithi189 ceph-mon[90992]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:48:47.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:47.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:48.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:48.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:48.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:48.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:48.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:48.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:48 smithi017 ceph-mon[103001]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 29724/61515 objects degraded (48.320%); 1254/61515 objects misplaced (2.039%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:48.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:48 smithi017 ceph-mon[107556]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 29724/61515 objects degraded (48.320%); 1254/61515 objects misplaced (2.039%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:48 smithi189 ceph-mon[90992]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 29724/61515 objects degraded (48.320%); 1254/61515 objects misplaced (2.039%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:48.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:48.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:48.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:49.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:49.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:49.151 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:49.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.297 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.399 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:49.421 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:49.493 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:49.493 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:49.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29724/61515 objects degraded (48.320%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:49.493 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29724/61515 objects degraded (48.320%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:49.523 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:49.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:49.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:49.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:49.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29724/61515 objects degraded (48.320%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:49.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:49.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:50.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:50.190 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:50.292 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:50.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:50.396 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:48:50.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:48:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:48:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:50 smithi189 ceph-mon[90992]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 29621/59322 objects degraded (49.933%); 1151/59322 objects misplaced (1.940%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1049193205' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:50 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:50.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:50.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[107556]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 29621/59322 objects degraded (49.933%); 1151/59322 objects misplaced (1.940%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1049193205' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[103001]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 29621/59322 objects degraded (49.933%); 1151/59322 objects misplaced (1.940%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1049193205' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:50 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:50.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:48:50.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:51.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:51.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:51.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:51.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:51.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:51.258 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:51.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:51.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:51.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:51.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:51.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:51.493 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1206,"stamp":"2024-09-13T14:48:45.282932+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1378978486,"num_objects":20999,"num_object_clones":0,"num_object_copies":62997,"num_objects_missing_on_primary":302,"num_objects_missing":302,"num_objects_degraded":29914,"num_objects_misplaced":1254,"num_objects_unfound":0,"num_objects_dirty":20999,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":162654,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28579,"num_bytes_recovered":1808959645,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127639,"ondisk_log_size":127639,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9922576,"kb_used_data":4548688,"kb_used_omap":44,"kb_used_meta":5373779,"kb_avail":740038640,"statfs":{"total":767960285184,"available":757799567360,"internally_reserved":0,"allocated":4657856512,"data_stored":4642135719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5502749943},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-234422272,"num_objects":-3577,"num_object_clones":0,"num_object_copies":-10731,"num_objects_missing_on_primary":-187,"num_objects_missing":-187,"num_objects_degraded":-613,"num_objects_misplaced":-189,"num_objects_unfound":0,"num_objects_dirty":-3577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3577,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":402,"num_bytes_recovered":26345472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.938898"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":621,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.349288+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:48:34.349288+0000","last_peered":"2024-09-13T14:48:34.349288+0000","last_clean":"2024-09-13T14:48:34.349288+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:48:34.349288+0000","last_undegraded":"2024-09-13T14:48:34.349288+0000","last_fullsized":"2024-09-13T14:48:34.349288+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":366,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360656+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:48:20.360656+0000","last_peered":"2024-09-13T14:48:20.360656+0000","last_clean":"2024-09-13T14:48:20.360656+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:48:20.360656+0000","last_undegraded":"2024-09-13T14:48:20.360656+0000","last_fullsized":"2024-09-13T14:48:20.360656+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":295,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339133+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:48:20.339133+0000","last_peered":"2024-09-13T14:48:20.339133+0000","last_clean":"2024-09-13T14:48:20.339133+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:48:20.339133+0000","last_undegraded":"2024-09-13T14:48:20.339133+0000","last_fullsized":"2024-09-13T14:48:20.339133+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":606,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.375289+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:48:34.375289+0000","last_peered":"2024-09-13T14:48:34.375289+0000","last_clean":"2024-09-13T14:48:34.375289+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:48:34.375289+0000","last_undegraded":"2024-09-13T14:48:34.375289+0000","last_fullsized":"2024-09-13T14:48:34.375289+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":607,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.375259+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:48:34.375259+0000","last_peered":"2024-09-13T14:48:34.375259+0000","last_clean":"2024-09-13T14:48:34.375259+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:48:34.375259+0000","last_undegraded":"2024-09-13T14:48:34.375259+0000","last_fullsized":"2024-09-13T14:48:34.375259+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":335,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360298+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:48:20.360298+0000","last_peered":"2024-09-13T14:48:20.360298+0000","last_clean":"2024-09-13T14:48:20.360298+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:48:20.360298+0000","last_undegraded":"2024-09-13T14:48:20.360298+0000","last_fullsized":"2024-09-13T14:48:20.360298+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":621,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348542+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:48:34.348542+0000","last_peered":"2024-09-13T14:48:34.348542+0000","last_clean":"2024-09-13T14:48:34.348542+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:48:34.348542+0000","last_undegraded":"2024-09-13T14:48:34.348542+0000","last_fullsized":"2024-09-13T14:48:34.348542+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":357,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339957+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:48:20.339957+0000","last_peered":"2024-09-13T14:48:20.339957+0000","last_clean":"2024-09-13T14:48:20.339957+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:48:20.339957+0000","last_undegraded":"2024-09-13T14:48:20.339957+0000","last_fullsized":"2024-09-13T14:48:20.339957+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"368'10021","reported_seq":23287,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:41.902416+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:48:41.902416+0000","last_peered":"2024-09-13T14:48:41.902416+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:48:41.902416+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"306'300","ondisk_log_start":"306'300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9721,"log_dups_size":0,"ondisk_log_size":9721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10021,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4376,"num_bytes_recovered":286785536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":883},{"shards":"2,6","objects":285}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":361,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.361069+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:48:20.361069+0000","last_peered":"2024-09-13T14:48:20.361069+0000","last_clean":"2024-09-13T14:48:20.361069+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:48:20.361069+0000","last_undegraded":"2024-09-13T14:48:20.361069+0000","last_fullsized":"2024-09-13T14:48:20.361069+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":280,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340112+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:48:20.340112+0000","last_peered":"2024-09-13T14:48:20.340112+0000","last_clean":"2024-09-13T14:48:20.340112+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:48:20.340112+0000","last_undegraded":"2024-09-13T14:48:20.340112+0000","last_fullsized":"2024-09-13T14:48:20.340112+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"368'10161","reported_seq":19726,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:42.545563+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:48:42.545563+0000","last_peered":"2024-09-13T14:48:42.545563+0000","last_clean":"2024-09-13T14:48:42.545563+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:48:42.545563+0000","last_undegraded":"2024-09-13T14:48:42.545563+0000","last_fullsized":"2024-09-13T14:48:42.545563+0000","mapping_epoch":326,"log_start":"307'2600","ondisk_log_start":"307'2600","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86966272,"num_objects":1327,"num_object_clones":0,"num_object_copies":3981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10161,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":379,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.363267+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:48:20.363267+0000","last_peered":"2024-09-13T14:48:20.363267+0000","last_clean":"2024-09-13T14:48:20.363267+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:48:20.363267+0000","last_undegraded":"2024-09-13T14:48:20.363267+0000","last_fullsized":"2024-09-13T14:48:20.363267+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344671+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:48:20.344671+0000","last_peered":"2024-09-13T14:48:20.344671+0000","last_clean":"2024-09-13T14:48:20.344671+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:48:20.344671+0000","last_undegraded":"2024-09-13T14:48:20.344671+0000","last_fullsized":"2024-09-13T14:48:20.344671+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"368'10484","reported_seq":10629,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:43.974832+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:48:43.974832+0000","last_peered":"2024-09-13T14:48:43.974832+0000","last_clean":"2024-09-13T14:48:43.974832+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:48:43.974832+0000","last_undegraded":"2024-09-13T14:48:43.974832+0000","last_fullsized":"2024-09-13T14:48:43.974832+0000","mapping_epoch":303,"log_start":"307'1900","ondisk_log_start":"307'1900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8584,"log_dups_size":0,"ondisk_log_size":8584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86114304,"num_objects":1314,"num_object_clones":0,"num_object_copies":3942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10484,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.361421+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:48:20.361421+0000","last_peered":"2024-09-13T14:48:20.361421+0000","last_clean":"2024-09-13T14:48:20.361421+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:48:20.361421+0000","last_undegraded":"2024-09-13T14:48:20.361421+0000","last_fullsized":"2024-09-13T14:48:20.361421+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":348,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339015+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:48:20.339015+0000","last_peered":"2024-09-13T14:48:20.339015+0000","last_clean":"2024-09-13T14:48:20.339015+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:48:20.339015+0000","last_undegraded":"2024-09-13T14:48:20.339015+0000","last_fullsized":"2024-09-13T14:48:20.339015+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"368'10275","reported_seq":10420,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:45.105268+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:48:45.105268+0000","last_peered":"2024-09-13T14:48:45.105268+0000","last_clean":"2024-09-13T14:48:45.105268+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:48:45.105268+0000","last_undegraded":"2024-09-13T14:48:45.105268+0000","last_fullsized":"2024-09-13T14:48:45.105268+0000","mapping_epoch":303,"log_start":"307'2700","ondisk_log_start":"307'2700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10275,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":296,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340920+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:48:20.340920+0000","last_peered":"2024-09-13T14:48:20.340920+0000","last_clean":"2024-09-13T14:48:20.340920+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:48:20.340920+0000","last_undegraded":"2024-09-13T14:48:20.340920+0000","last_fullsized":"2024-09-13T14:48:20.340920+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":633,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.374531+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:48:34.374531+0000","last_peered":"2024-09-13T14:48:34.374531+0000","last_clean":"2024-09-13T14:48:34.374531+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:48:34.374531+0000","last_undegraded":"2024-09-13T14:48:34.374531+0000","last_fullsized":"2024-09-13T14:48:34.374531+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"368'10133","reported_seq":10275,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:40.681318+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:48:40.681318+0000","last_peered":"2024-09-13T14:48:40.681318+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:48:40.681318+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"307'3100","ondisk_log_start":"307'3100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87097344,"num_objects":1329,"num_object_clones":0,"num_object_copies":3987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10133,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-4304}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":479,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360082+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:48:20.360082+0000","last_peered":"2024-09-13T14:48:20.360082+0000","last_clean":"2024-09-13T14:48:20.360082+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:48:20.360082+0000","last_undegraded":"2024-09-13T14:48:20.360082+0000","last_fullsized":"2024-09-13T14:48:20.360082+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":630,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348100+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:48:34.348100+0000","last_peered":"2024-09-13T14:48:34.348100+0000","last_clean":"2024-09-13T14:48:34.348100+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:48:34.348100+0000","last_undegraded":"2024-09-13T14:48:34.348100+0000","last_fullsized":"2024-09-13T14:48:34.348100+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344716+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:48:20.344716+0000","last_peered":"2024-09-13T14:48:20.344716+0000","last_clean":"2024-09-13T14:48:20.344716+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:48:20.344716+0000","last_undegraded":"2024-09-13T14:48:20.344716+0000","last_fullsized":"2024-09-13T14:48:20.344716+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"368'9885","reported_seq":10030,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:45.064964+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:48:45.064964+0000","last_peered":"2024-09-13T14:48:45.064964+0000","last_clean":"2024-09-13T14:48:45.064964+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:48:45.064964+0000","last_undegraded":"2024-09-13T14:48:45.064964+0000","last_fullsized":"2024-09-13T14:48:45.064964+0000","mapping_epoch":303,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7585,"log_dups_size":0,"ondisk_log_size":7585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9885,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":183,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339978+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:48:20.339978+0000","last_peered":"2024-09-13T14:48:20.339978+0000","last_clean":"2024-09-13T14:48:20.339978+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:48:20.339978+0000","last_undegraded":"2024-09-13T14:48:20.339978+0000","last_fullsized":"2024-09-13T14:48:20.339978+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":369,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.338896+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:48:20.338896+0000","last_peered":"2024-09-13T14:48:20.338896+0000","last_clean":"2024-09-13T14:48:20.338896+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:48:20.338896+0000","last_undegraded":"2024-09-13T14:48:20.338896+0000","last_fullsized":"2024-09-13T14:48:20.338896+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":81,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.345029+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:48:20.345029+0000","last_peered":"2024-09-13T14:48:20.345029+0000","last_clean":"2024-09-13T14:48:20.345029+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:48:20.345029+0000","last_undegraded":"2024-09-13T14:48:20.345029+0000","last_fullsized":"2024-09-13T14:48:20.345029+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"368'10054","reported_seq":22828,"reported_epoch":368,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:42.698753+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:48:42.698753+0000","last_peered":"2024-09-13T14:48:42.698753+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:48:42.698753+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'2700","ondisk_log_start":"307'2700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":302,"num_objects_missing":302,"num_objects_degraded":677,"num_objects_misplaced":1254,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10054,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4349,"num_bytes_recovered":285016064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":302},{"shards":"0,3,4","objects":952},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344514+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:48:20.344514+0000","last_peered":"2024-09-13T14:48:20.344514+0000","last_clean":"2024-09-13T14:48:20.344514+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:48:20.344514+0000","last_undegraded":"2024-09-13T14:48:20.344514+0000","last_fullsized":"2024-09-13T14:48:20.344514+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344953+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:48:20.344953+0000","last_peered":"2024-09-13T14:48:20.344953+0000","last_clean":"2024-09-13T14:48:20.344953+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:48:20.344953+0000","last_undegraded":"2024-09-13T14:48:20.344953+0000","last_fullsized":"2024-09-13T14:48:20.344953+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"368'9895","reported_seq":10040,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:44.994277+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:48:44.994277+0000","last_peered":"2024-09-13T14:48:44.994277+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:48:44.994277+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9895,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-4203}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":176,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339334+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:48:20.339334+0000","last_peered":"2024-09-13T14:48:20.339334+0000","last_clean":"2024-09-13T14:48:20.339334+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:48:20.339334+0000","last_undegraded":"2024-09-13T14:48:20.339334+0000","last_fullsized":"2024-09-13T14:48:20.339334+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":623,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.360564+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:48:34.360564+0000","last_peered":"2024-09-13T14:48:34.360564+0000","last_clean":"2024-09-13T14:48:34.360564+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:48:34.360564+0000","last_undegraded":"2024-09-13T14:48:34.360564+0000","last_fullsized":"2024-09-13T14:48:34.360564+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":585,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.356611+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:48:34.356611+0000","last_peered":"2024-09-13T14:48:34.356611+0000","last_clean":"2024-09-13T14:48:34.356611+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:48:34.356611+0000","last_undegraded":"2024-09-13T14:48:34.356611+0000","last_fullsized":"2024-09-13T14:48:34.356611+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"368'9907","reported_seq":9987,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:42.540727+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:48:42.540727+0000","last_peered":"2024-09-13T14:48:42.540727+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:48:42.540727+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7507,"log_dups_size":0,"ondisk_log_size":7507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9907,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-4183}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":470,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360220+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:48:20.360220+0000","last_peered":"2024-09-13T14:48:20.360220+0000","last_clean":"2024-09-13T14:48:20.360220+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:48:20.360220+0000","last_undegraded":"2024-09-13T14:48:20.360220+0000","last_fullsized":"2024-09-13T14:48:20.360220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":892,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339221+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:48:20.339221+0000","last_peered":"2024-09-13T14:48:20.339221+0000","last_clean":"2024-09-13T14:48:20.339221+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:48:20.339221+0000","last_undegraded":"2024-09-13T14:48:20.339221+0000","last_fullsized":"2024-09-13T14:48:20.339221+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":576,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348823+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:48:34.348823+0000","last_peered":"2024-09-13T14:48:34.348823+0000","last_clean":"2024-09-13T14:48:34.348823+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:48:34.348823+0000","last_undegraded":"2024-09-13T14:48:34.348823+0000","last_fullsized":"2024-09-13T14:48:34.348823+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":334,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340676+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:48:20.340676+0000","last_peered":"2024-09-13T14:48:20.340676+0000","last_clean":"2024-09-13T14:48:20.340676+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:48:20.340676+0000","last_undegraded":"2024-09-13T14:48:20.340676+0000","last_fullsized":"2024-09-13T14:48:20.340676+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"368'10246","reported_seq":10389,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:45.060141+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:48:45.060141+0000","last_peered":"2024-09-13T14:48:45.060141+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:48:45.060141+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'2700","ondisk_log_start":"307'2700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10246,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-4378}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":430,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.356228+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:48:34.356228+0000","last_peered":"2024-09-13T14:48:34.356228+0000","last_clean":"2024-09-13T14:48:34.356228+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:48:34.356228+0000","last_undegraded":"2024-09-13T14:48:34.356228+0000","last_fullsized":"2024-09-13T14:48:34.356228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":758,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360138+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:48:20.360138+0000","last_peered":"2024-09-13T14:48:20.360138+0000","last_clean":"2024-09-13T14:48:20.360138+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:48:20.360138+0000","last_undegraded":"2024-09-13T14:48:20.360138+0000","last_fullsized":"2024-09-13T14:48:20.360138+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":626,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.374664+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:48:34.374664+0000","last_peered":"2024-09-13T14:48:34.374664+0000","last_clean":"2024-09-13T14:48:34.374664+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:48:34.374664+0000","last_undegraded":"2024-09-13T14:48:34.374664+0000","last_fullsized":"2024-09-13T14:48:34.374664+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":630,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.375435+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:48:34.375435+0000","last_peered":"2024-09-13T14:48:34.375435+0000","last_clean":"2024-09-13T14:48:34.375435+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:48:34.375435+0000","last_undegraded":"2024-09-13T14:48:34.375435+0000","last_fullsized":"2024-09-13T14:48:34.375435+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"368'10248","reported_seq":19934,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:40.726744+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:48:40.726744+0000","last_peered":"2024-09-13T14:48:40.726744+0000","last_clean":"2024-09-13T14:48:40.726744+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:48:40.726744+0000","last_undegraded":"2024-09-13T14:48:40.726744+0000","last_fullsized":"2024-09-13T14:48:40.726744+0000","mapping_epoch":330,"log_start":"307'3200","ondisk_log_start":"307'3200","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92798976,"num_objects":1416,"num_object_clones":0,"num_object_copies":4248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10248,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":499,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.360395+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:48:20.360395+0000","last_peered":"2024-09-13T14:48:20.360395+0000","last_clean":"2024-09-13T14:48:20.360395+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:48:20.360395+0000","last_undegraded":"2024-09-13T14:48:20.360395+0000","last_fullsized":"2024-09-13T14:48:20.360395+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":594,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348646+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:48:34.348646+0000","last_peered":"2024-09-13T14:48:34.348646+0000","last_clean":"2024-09-13T14:48:34.348646+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:48:34.348646+0000","last_undegraded":"2024-09-13T14:48:34.348646+0000","last_fullsized":"2024-09-13T14:48:34.348646+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":563,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.354603+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:48:34.354603+0000","last_peered":"2024-09-13T14:48:34.354603+0000","last_clean":"2024-09-13T14:48:34.354603+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:48:34.354603+0000","last_undegraded":"2024-09-13T14:48:34.354603+0000","last_fullsized":"2024-09-13T14:48:34.354603+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.338905+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:48:20.338905+0000","last_peered":"2024-09-13T14:48:20.338905+0000","last_clean":"2024-09-13T14:48:20.338905+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:48:20.338905+0000","last_undegraded":"2024-09-13T14:48:20.338905+0000","last_fullsized":"2024-09-13T14:48:20.338905+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"368'9891","reported_seq":10035,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:40.750656+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:48:40.750656+0000","last_peered":"2024-09-13T14:48:40.750656+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:48:40.750656+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"307'2900","ondisk_log_start":"307'2900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9891,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-4170}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":176,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.338969+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:48:20.338969+0000","last_peered":"2024-09-13T14:48:20.338969+0000","last_clean":"2024-09-13T14:48:20.338969+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:48:20.338969+0000","last_undegraded":"2024-09-13T14:48:20.338969+0000","last_fullsized":"2024-09-13T14:48:20.338969+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":983,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339413+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:48:20.339413+0000","last_peered":"2024-09-13T14:48:20.339413+0000","last_clean":"2024-09-13T14:48:20.339413+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:48:20.339413+0000","last_undegraded":"2024-09-13T14:48:20.339413+0000","last_fullsized":"2024-09-13T14:48:20.339413+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.346583+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:48:20.346583+0000","last_peered":"2024-09-13T14:48:20.346583+0000","last_clean":"2024-09-13T14:48:20.346583+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:48:20.346583+0000","last_undegraded":"2024-09-13T14:48:20.346583+0000","last_fullsized":"2024-09-13T14:48:20.346583+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":583,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.354233+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:48:34.354233+0000","last_peered":"2024-09-13T14:48:34.354233+0000","last_clean":"2024-09-13T14:48:34.354233+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:48:34.354233+0000","last_undegraded":"2024-09-13T14:48:34.354233+0000","last_fullsized":"2024-09-13T14:48:34.354233+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"368'10027","reported_seq":19452,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:44.061642+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:48:44.061642+0000","last_peered":"2024-09-13T14:48:44.061642+0000","last_clean":"2024-09-13T14:48:44.061642+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:48:44.061642+0000","last_undegraded":"2024-09-13T14:48:44.061642+0000","last_fullsized":"2024-09-13T14:48:44.061642+0000","mapping_epoch":337,"log_start":"307'1400","ondisk_log_start":"307'1400","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87359488,"num_objects":1333,"num_object_clones":0,"num_object_copies":3999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10027,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":367,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340235+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:48:20.340235+0000","last_peered":"2024-09-13T14:48:20.340235+0000","last_clean":"2024-09-13T14:48:20.340235+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:48:20.340235+0000","last_undegraded":"2024-09-13T14:48:20.340235+0000","last_fullsized":"2024-09-13T14:48:20.340235+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":313,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339253+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:48:20.339253+0000","last_peered":"2024-09-13T14:48:20.339253+0000","last_clean":"2024-09-13T14:48:20.339253+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:48:20.339253+0000","last_undegraded":"2024-09-13T14:48:20.339253+0000","last_fullsized":"2024-09-13T14:48:20.339253+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1162,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.354451+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:48:34.354451+0000","last_peered":"2024-09-13T14:48:34.354451+0000","last_clean":"2024-09-13T14:48:34.354451+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:48:34.354451+0000","last_undegraded":"2024-09-13T14:48:34.354451+0000","last_fullsized":"2024-09-13T14:48:34.354451+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":564,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.365783+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:48:34.365783+0000","last_peered":"2024-09-13T14:48:34.365783+0000","last_clean":"2024-09-13T14:48:34.365783+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:48:34.365783+0000","last_undegraded":"2024-09-13T14:48:34.365783+0000","last_fullsized":"2024-09-13T14:48:34.365783+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"368'9855","reported_seq":10000,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:40.642866+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:48:40.642866+0000","last_peered":"2024-09-13T14:48:40.642866+0000","last_clean":"2024-09-13T14:48:40.642866+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:48:40.642866+0000","last_undegraded":"2024-09-13T14:48:40.642866+0000","last_fullsized":"2024-09-13T14:48:40.642866+0000","mapping_epoch":303,"log_start":"307'2800","ondisk_log_start":"307'2800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9855,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":367,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339527+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:48:20.339527+0000","last_peered":"2024-09-13T14:48:20.339527+0000","last_clean":"2024-09-13T14:48:20.339527+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:48:20.339527+0000","last_undegraded":"2024-09-13T14:48:20.339527+0000","last_fullsized":"2024-09-13T14:48:20.339527+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":573,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.354358+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:48:34.354358+0000","last_peered":"2024-09-13T14:48:34.354358+0000","last_clean":"2024-09-13T14:48:34.354358+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:48:34.354358+0000","last_undegraded":"2024-09-13T14:48:34.354358+0000","last_fullsized":"2024-09-13T14:48:34.354358+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":762,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.361427+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:48:20.361427+0000","last_peered":"2024-09-13T14:48:20.361427+0000","last_clean":"2024-09-13T14:48:20.361427+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:48:20.361427+0000","last_undegraded":"2024-09-13T14:48:20.361427+0000","last_fullsized":"2024-09-13T14:48:20.361427+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":609,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.374444+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:48:34.374444+0000","last_peered":"2024-09-13T14:48:34.374444+0000","last_clean":"2024-09-13T14:48:34.374444+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:48:34.374444+0000","last_undegraded":"2024-09-13T14:48:34.374444+0000","last_fullsized":"2024-09-13T14:48:34.374444+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"368'10389","reported_seq":10534,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:43.980754+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:48:43.980754+0000","last_peered":"2024-09-13T14:48:43.980754+0000","last_clean":"2024-09-13T14:48:43.980754+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:48:43.980754+0000","last_undegraded":"2024-09-13T14:48:43.980754+0000","last_fullsized":"2024-09-13T14:48:43.980754+0000","mapping_epoch":303,"log_start":"307'1800","ondisk_log_start":"307'1800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8589,"log_dups_size":0,"ondisk_log_size":8589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85262336,"num_objects":1301,"num_object_clones":0,"num_object_copies":3903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10389,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344951+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:48:20.344951+0000","last_peered":"2024-09-13T14:48:20.344951+0000","last_clean":"2024-09-13T14:48:20.344951+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:48:20.344951+0000","last_undegraded":"2024-09-13T14:48:20.344951+0000","last_fullsized":"2024-09-13T14:48:20.344951+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":609,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.352941+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:48:34.352941+0000","last_peered":"2024-09-13T14:48:34.352941+0000","last_clean":"2024-09-13T14:48:34.352941+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:48:34.352941+0000","last_undegraded":"2024-09-13T14:48:34.352941+0000","last_fullsized":"2024-09-13T14:48:34.352941+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":949,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.355099+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:48:34.355099+0000","last_peered":"2024-09-13T14:48:34.355099+0000","last_clean":"2024-09-13T14:48:34.355099+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:48:34.355099+0000","last_undegraded":"2024-09-13T14:48:34.355099+0000","last_fullsized":"2024-09-13T14:48:34.355099+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":607,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.360461+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:48:34.360461+0000","last_peered":"2024-09-13T14:48:34.360461+0000","last_clean":"2024-09-13T14:48:34.360461+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:48:34.360461+0000","last_undegraded":"2024-09-13T14:48:34.360461+0000","last_fullsized":"2024-09-13T14:48:34.360461+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"368'10236","reported_seq":21615,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:42.842787+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:48:42.842787+0000","last_peered":"2024-09-13T14:48:42.842787+0000","last_clean":"2024-09-13T14:48:42.842787+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:48:42.842787+0000","last_undegraded":"2024-09-13T14:48:42.842787+0000","last_fullsized":"2024-09-13T14:48:42.842787+0000","mapping_epoch":332,"log_start":"306'1100","ondisk_log_start":"306'1100","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9136,"log_dups_size":0,"ondisk_log_size":9136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689496,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10236,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":460,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.349121+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:48:34.349121+0000","last_peered":"2024-09-13T14:48:34.349121+0000","last_clean":"2024-09-13T14:48:34.349121+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:48:34.349121+0000","last_undegraded":"2024-09-13T14:48:34.349121+0000","last_fullsized":"2024-09-13T14:48:34.349121+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":644,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.375150+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:48:34.375150+0000","last_peered":"2024-09-13T14:48:34.375150+0000","last_clean":"2024-09-13T14:48:34.375150+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:48:34.375150+0000","last_undegraded":"2024-09-13T14:48:34.375150+0000","last_fullsized":"2024-09-13T14:48:34.375150+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":786,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340394+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:48:20.340394+0000","last_peered":"2024-09-13T14:48:20.340394+0000","last_clean":"2024-09-13T14:48:20.340394+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:48:20.340394+0000","last_undegraded":"2024-09-13T14:48:20.340394+0000","last_fullsized":"2024-09-13T14:48:20.340394+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344613+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:48:20.344613+0000","last_peered":"2024-09-13T14:48:20.344613+0000","last_clean":"2024-09-13T14:48:20.344613+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:48:20.344613+0000","last_undegraded":"2024-09-13T14:48:20.344613+0000","last_fullsized":"2024-09-13T14:48:20.344613+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.346648+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:48:20.346648+0000","last_peered":"2024-09-13T14:48:20.346648+0000","last_clean":"2024-09-13T14:48:20.346648+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:48:20.346648+0000","last_undegraded":"2024-09-13T14:48:20.346648+0000","last_fullsized":"2024-09-13T14:48:20.346648+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":77,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344640+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:48:20.344640+0000","last_peered":"2024-09-13T14:48:20.344640+0000","last_clean":"2024-09-13T14:48:20.344640+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:48:20.344640+0000","last_undegraded":"2024-09-13T14:48:20.344640+0000","last_fullsized":"2024-09-13T14:48:20.344640+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":561,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.365183+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:48:34.365183+0000","last_peered":"2024-09-13T14:48:34.365183+0000","last_clean":"2024-09-13T14:48:34.365183+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:48:34.365183+0000","last_undegraded":"2024-09-13T14:48:34.365183+0000","last_fullsized":"2024-09-13T14:48:34.365183+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":367,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.361343+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:48:20.361343+0000","last_peered":"2024-09-13T14:48:20.361343+0000","last_clean":"2024-09-13T14:48:20.361343+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:48:20.361343+0000","last_undegraded":"2024-09-13T14:48:20.361343+0000","last_fullsized":"2024-09-13T14:48:20.361343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.338748+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:48:20.338748+0000","last_peered":"2024-09-13T14:48:20.338748+0000","last_clean":"2024-09-13T14:48:20.338748+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:48:20.338748+0000","last_undegraded":"2024-09-13T14:48:20.338748+0000","last_fullsized":"2024-09-13T14:48:20.338748+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":565,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348395+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:48:34.348395+0000","last_peered":"2024-09-13T14:48:34.348395+0000","last_clean":"2024-09-13T14:48:34.348395+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:48:34.348395+0000","last_undegraded":"2024-09-13T14:48:34.348395+0000","last_fullsized":"2024-09-13T14:48:34.348395+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":617,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.373986+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:48:34.373986+0000","last_peered":"2024-09-13T14:48:34.373986+0000","last_clean":"2024-09-13T14:48:34.373986+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:48:34.373986+0000","last_undegraded":"2024-09-13T14:48:34.373986+0000","last_fullsized":"2024-09-13T14:48:34.373986+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":621,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.360318+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:48:34.360318+0000","last_peered":"2024-09-13T14:48:34.360318+0000","last_clean":"2024-09-13T14:48:34.360318+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:48:34.360318+0000","last_undegraded":"2024-09-13T14:48:34.360318+0000","last_fullsized":"2024-09-13T14:48:34.360318+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":365,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340309+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:48:20.340309+0000","last_peered":"2024-09-13T14:48:20.340309+0000","last_clean":"2024-09-13T14:48:20.340309+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:48:20.340309+0000","last_undegraded":"2024-09-13T14:48:20.340309+0000","last_fullsized":"2024-09-13T14:48:20.340309+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":639,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.360491+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:48:34.360491+0000","last_peered":"2024-09-13T14:48:34.360491+0000","last_clean":"2024-09-13T14:48:34.360491+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:48:34.360491+0000","last_undegraded":"2024-09-13T14:48:34.360491+0000","last_fullsized":"2024-09-13T14:48:34.360491+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":609,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.375345+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:48:34.375345+0000","last_peered":"2024-09-13T14:48:34.375345+0000","last_clean":"2024-09-13T14:48:34.375345+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:48:34.375345+0000","last_undegraded":"2024-09-13T14:48:34.375345+0000","last_fullsized":"2024-09-13T14:48:34.375345+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":585,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.355409+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:48:34.355409+0000","last_peered":"2024-09-13T14:48:34.355409+0000","last_clean":"2024-09-13T14:48:34.355409+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:48:34.355409+0000","last_undegraded":"2024-09-13T14:48:34.355409+0000","last_fullsized":"2024-09-13T14:48:34.355409+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":634,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348223+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:48:34.348223+0000","last_peered":"2024-09-13T14:48:34.348223+0000","last_clean":"2024-09-13T14:48:34.348223+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:48:34.348223+0000","last_undegraded":"2024-09-13T14:48:34.348223+0000","last_fullsized":"2024-09-13T14:48:34.348223+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":66,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.353541+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:48:20.353541+0000","last_peered":"2024-09-13T14:48:20.353541+0000","last_clean":"2024-09-13T14:48:20.353541+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:48:20.353541+0000","last_undegraded":"2024-09-13T14:48:20.353541+0000","last_fullsized":"2024-09-13T14:48:20.353541+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":531,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.354635+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:48:34.354635+0000","last_peered":"2024-09-13T14:48:34.354635+0000","last_clean":"2024-09-13T14:48:34.354635+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:48:34.354635+0000","last_undegraded":"2024-09-13T14:48:34.354635+0000","last_fullsized":"2024-09-13T14:48:34.354635+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":612,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.374309+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:48:34.374309+0000","last_peered":"2024-09-13T14:48:34.374309+0000","last_clean":"2024-09-13T14:48:34.374309+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:48:34.374309+0000","last_undegraded":"2024-09-13T14:48:34.374309+0000","last_fullsized":"2024-09-13T14:48:34.374309+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.346605+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:48:20.346605+0000","last_peered":"2024-09-13T14:48:20.346605+0000","last_clean":"2024-09-13T14:48:20.346605+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:48:20.346605+0000","last_undegraded":"2024-09-13T14:48:20.346605+0000","last_fullsized":"2024-09-13T14:48:20.346605+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.340937+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:48:20.340937+0000","last_peered":"2024-09-13T14:48:20.340937+0000","last_clean":"2024-09-13T14:48:20.340937+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:48:20.340937+0000","last_undegraded":"2024-09-13T14:48:20.340937+0000","last_fullsized":"2024-09-13T14:48:20.340937+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.353495+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:48:20.353495+0000","last_peered":"2024-09-13T14:48:20.353495+0000","last_clean":"2024-09-13T14:48:20.353495+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:48:20.353495+0000","last_undegraded":"2024-09-13T14:48:20.353495+0000","last_fullsized":"2024-09-13T14:48:20.353495+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":299,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.339209+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:48:20.339209+0000","last_peered":"2024-09-13T14:48:20.339209+0000","last_clean":"2024-09-13T14:48:20.339209+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:48:20.339209+0000","last_undegraded":"2024-09-13T14:48:20.339209+0000","last_fullsized":"2024-09-13T14:48:20.339209+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":631,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348735+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:48:34.348735+0000","last_peered":"2024-09-13T14:48:34.348735+0000","last_clean":"2024-09-13T14:48:34.348735+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:48:34.348735+0000","last_undegraded":"2024-09-13T14:48:34.348735+0000","last_fullsized":"2024-09-13T14:48:34.348735+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":618,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348670+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:48:34.348670+0000","last_peered":"2024-09-13T14:48:34.348670+0000","last_clean":"2024-09-13T14:48:34.348670+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:48:34.348670+0000","last_undegraded":"2024-09-13T14:48:34.348670+0000","last_fullsized":"2024-09-13T14:48:34.348670+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-09-13T14:48:20.344476+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:48:20.344476+0000","last_peered":"2024-09-13T14:48:20.344476+0000","last_clean":"2024-09-13T14:48:20.344476+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:48:20.344476+0000","last_undegraded":"2024-09-13T14:48:20.344476+0000","last_fullsized":"2024-09-13T14:48:20.344476+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":623,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.373937+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:48:34.373937+0000","last_peered":"2024-09-13T14:48:34.373937+0000","last_clean":"2024-09-13T14:48:34.373937+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:48:34.373937+0000","last_undegraded":"2024-09-13T14:48:34.373937+0000","last_fullsized":"2024-09-13T14:48:34.373937+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1374289944,"num_objects":20971,"num_object_clones":0,"num_object_copies":62913,"num_objects_missing_on_primary":302,"num_objects_missing":302,"num_objects_degraded":29914,"num_objects_misplaced":1254,"num_objects_unfound":0,"num_objects_dirty":20971,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161707,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28522,"num_bytes_recovered":1808793600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4578619392,"data_stored":4578607176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125507,"ondisk_log_size":125507,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397951,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1327932,"kb_used_data":431924,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":92417220,"statfs":{"total":95995035648,"available":94635233280,"internally_reserved":0,"allocated":442290176,"data_stored":440309521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594179,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1678632,"kb_used_data":784160,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":92066520,"statfs":{"total":95995035648,"available":94276116480,"internally_reserved":0,"allocated":802979840,"data_stored":801018939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665633,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462952,"kb_used_data":134560,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93282200,"statfs":{"total":95995035648,"available":95520972800,"internally_reserved":0,"allocated":137789440,"data_stored":135824870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986635,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1298100,"kb_used_data":398252,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92447052,"statfs":{"total":95995035648,"available":94665781248,"internally_reserved":0,"allocated":407810048,"data_stored":405837491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697194,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646884,"kb_used_data":888732,"kb_used_omap":4,"kb_used_meta":758139,"kb_avail":92098268,"statfs":{"total":95995035648,"available":94308626432,"internally_reserved":0,"allocated":910061568,"data_stored":908101151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":776334411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860721,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458092,"kb_used_data":818916,"kb_used_omap":8,"kb_used_meta":639159,"kb_avail":92287060,"statfs":{"total":95995035648,"available":94501949440,"internally_reserved":0,"allocated":838569984,"data_stored":836601913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":654499660},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020393,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907296,"kb_used_data":581528,"kb_used_omap":9,"kb_used_meta":325750,"kb_avail":92837856,"statfs":{"total":95995035648,"available":95065964544,"internally_reserved":0,"allocated":595484672,"data_stored":593527728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":333568837},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187783,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1142688,"kb_used_data":510616,"kb_used_omap":7,"kb_used_meta":632056,"kb_avail":92602464,"statfs":{"total":95995035648,"available":94824923136,"internally_reserved":0,"allocated":522870784,"data_stored":520914106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":647225675},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":510918656,"data_stored":510918656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":582483968,"data_stored":582483968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":827330560,"data_stored":827326488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":900792320,"data_stored":900792320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":398917632,"data_stored":398917632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":131469312,"data_stored":131465240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":796327936,"data_stored":796327936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":430379008,"data_stored":430374936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:48:51.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.596 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[107556]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:51 smithi017 ceph-mon[103001]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:48:51.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:51 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:51.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:51 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:51.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:51 smithi189 ceph-mon[90992]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:48:51.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:51.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:51.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:51.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:52.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.146 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:48:52.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:52.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:48:52.169 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:52.191 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:48:52.191 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:52.208 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:48:52.209 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:52.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:52.226 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:48:52.226 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:52.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:52.249 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:48:52.249 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:52.268 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:48:52.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:52.287 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:48:52.288 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:52.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:52.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:52.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:52.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:52.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:52 smithi017 ceph-mon[103001]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 29621/56676 objects degraded (52.264%); 1151/56676 objects misplaced (2.031%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:52.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3605238838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:52.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:52 smithi017 ceph-mon[107556]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 29621/56676 objects degraded (52.264%); 1151/56676 objects misplaced (2.031%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:52.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3605238838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:52.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:52 smithi189 ceph-mon[90992]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 29621/56676 objects degraded (52.264%); 1151/56676 objects misplaced (2.031%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:48:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3605238838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:52.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:52.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:52.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:53.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:53.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:53.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.268 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:53.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:53.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.934 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:53.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.971 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:48:53.972 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:48:53.972 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:48:53.972 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:48:54.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.057 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:54.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:54.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:54.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:54.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:54.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:54.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:48:54.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:54.671 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:54 smithi189 ceph-mon[90992]: pgmap v1211: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 29304/54975 objects degraded (53.304%); 1043/54975 objects misplaced (1.897%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:54.671 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29304/54975 objects degraded (53.304%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:54.672 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.695 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:48:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:54 smithi017 ceph-mon[103001]: pgmap v1211: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 29304/54975 objects degraded (53.304%); 1043/54975 objects misplaced (1.897%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29304/54975 objects degraded (53.304%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:54 smithi017 ceph-mon[107556]: pgmap v1211: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 29304/54975 objects degraded (53.304%); 1043/54975 objects misplaced (1.897%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29304/54975 objects degraded (53.304%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:54.795 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:54.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:54.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:54.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:55.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:55.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:55.061 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:55.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:55.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:55.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:55.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:55.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:55.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:55.555 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:55.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:48:55.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:55.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:55.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:55.944 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:55.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:55.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:56.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:56.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:56.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:56.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:56.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:56.349 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:56.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:56.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:56.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.602 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:56 smithi017 ceph-mon[103001]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 29304/52773 objects degraded (55.528%); 1043/52773 objects misplaced (1.976%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:56 smithi017 ceph-mon[107556]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 29304/52773 objects degraded (55.528%); 1043/52773 objects misplaced (1.976%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:56.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:56 smithi189 ceph-mon[90992]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 29304/52773 objects degraded (55.528%); 1043/52773 objects misplaced (1.976%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:48:56.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:56.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:56.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:57.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:57.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:57.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:57.310 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:48:57.310 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:48:57.310 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:48:57.310 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:48:57.311 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:48:57.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:48:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:48:57] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-13T14:48:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:57.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:57.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:57.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:57.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:57.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:57.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:58.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:58.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:58.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:58.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:48:58.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.487 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:58 smithi189 ceph-mon[90992]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 29132/51162 objects degraded (56.941%); 1043/51162 objects misplaced (2.039%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:58.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:58.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:58.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:58.692 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:58 smithi017 ceph-mon[107556]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 29132/51162 objects degraded (56.941%); 1043/51162 objects misplaced (2.039%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:58 smithi017 ceph-mon[103001]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 29132/51162 objects degraded (56.941%); 1043/51162 objects misplaced (2.039%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:48:58.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:58.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:58.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:58.926 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:48:58.928 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:48:58.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:59.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:59.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:59.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:59.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:59.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:59.483 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:59.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:48:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29132/51162 objects degraded (56.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:48:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29132/51162 objects degraded (56.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:48:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29132/51162 objects degraded (56.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:48:59.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:48:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.910 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:59.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:48:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:00.060 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1213,"stamp":"2024-09-13T14:48:57.286172+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1120438966,"num_objects":17054,"num_object_clones":0,"num_object_copies":51162,"num_objects_missing_on_primary":90,"num_objects_missing":90,"num_objects_degraded":29132,"num_objects_misplaced":1043,"num_objects_unfound":0,"num_objects_dirty":17054,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":166599,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29075,"num_bytes_recovered":1841465501,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127684,"ondisk_log_size":127684,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9304792,"kb_used_data":3934992,"kb_used_omap":44,"kb_used_meta":5369619,"kb_avail":740656424,"statfs":{"total":767960285184,"available":758432178176,"internally_reserved":0,"allocated":4029431808,"data_stored":4013648239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5498490103},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-226164736,"num_objects":-3451,"num_object_clones":0,"num_object_copies":-10353,"num_objects_missing_on_primary":-212,"num_objects_missing":-212,"num_objects_degraded":-592,"num_objects_misplaced":-211,"num_objects_unfound":0,"num_objects_dirty":-3451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3451,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":401,"num_bytes_recovered":26279936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002647"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":623,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402277+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:48:50.402277+0000","last_peered":"2024-09-13T14:48:50.402277+0000","last_clean":"2024-09-13T14:48:50.402277+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:48:50.402277+0000","last_undegraded":"2024-09-13T14:48:50.402277+0000","last_fullsized":"2024-09-13T14:48:50.402277+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":367,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.363167+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:48:34.363167+0000","last_peered":"2024-09-13T14:48:34.363167+0000","last_clean":"2024-09-13T14:48:34.363167+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:48:34.363167+0000","last_undegraded":"2024-09-13T14:48:34.363167+0000","last_fullsized":"2024-09-13T14:48:34.363167+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":296,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348867+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:48:34.348867+0000","last_peered":"2024-09-13T14:48:34.348867+0000","last_clean":"2024-09-13T14:48:34.348867+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:48:34.348867+0000","last_undegraded":"2024-09-13T14:48:34.348867+0000","last_fullsized":"2024-09-13T14:48:34.348867+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":608,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414650+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:48:50.414650+0000","last_peered":"2024-09-13T14:48:50.414650+0000","last_clean":"2024-09-13T14:48:50.414650+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:48:50.414650+0000","last_undegraded":"2024-09-13T14:48:50.414650+0000","last_fullsized":"2024-09-13T14:48:50.414650+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":609,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.415067+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:48:50.415067+0000","last_peered":"2024-09-13T14:48:50.415067+0000","last_clean":"2024-09-13T14:48:50.415067+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:48:50.415067+0000","last_undegraded":"2024-09-13T14:48:50.415067+0000","last_fullsized":"2024-09-13T14:48:50.415067+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":336,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.362963+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:48:34.362963+0000","last_peered":"2024-09-13T14:48:34.362963+0000","last_clean":"2024-09-13T14:48:34.362963+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:48:34.362963+0000","last_undegraded":"2024-09-13T14:48:34.362963+0000","last_fullsized":"2024-09-13T14:48:34.362963+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":623,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402593+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:48:50.402593+0000","last_peered":"2024-09-13T14:48:50.402593+0000","last_clean":"2024-09-13T14:48:50.402593+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:48:50.402593+0000","last_undegraded":"2024-09-13T14:48:50.402593+0000","last_fullsized":"2024-09-13T14:48:50.402593+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":358,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.345990+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:48:34.345990+0000","last_peered":"2024-09-13T14:48:34.345990+0000","last_clean":"2024-09-13T14:48:34.345990+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:48:34.345990+0000","last_undegraded":"2024-09-13T14:48:34.345990+0000","last_fullsized":"2024-09-13T14:48:34.345990+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"369'10329","reported_seq":24453,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:57.088009+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:48:57.088009+0000","last_peered":"2024-09-13T14:48:57.088009+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:48:57.088009+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"306'600","ondisk_log_start":"306'600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9729,"log_dups_size":0,"ondisk_log_size":9729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62980096,"num_objects":961,"num_object_clones":0,"num_object_copies":2883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10329,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4661,"num_bytes_recovered":305463296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":78},{"shards":"2,4,6","objects":860}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":362,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.363313+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:48:34.363313+0000","last_peered":"2024-09-13T14:48:34.363313+0000","last_clean":"2024-09-13T14:48:34.363313+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:48:34.363313+0000","last_undegraded":"2024-09-13T14:48:34.363313+0000","last_fullsized":"2024-09-13T14:48:34.363313+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":281,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.351020+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:48:34.351020+0000","last_peered":"2024-09-13T14:48:34.351020+0000","last_clean":"2024-09-13T14:48:34.351020+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:48:34.351020+0000","last_undegraded":"2024-09-13T14:48:34.351020+0000","last_fullsized":"2024-09-13T14:48:34.351020+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"369'10398","reported_seq":19964,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:52.594764+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:48:52.594764+0000","last_peered":"2024-09-13T14:48:52.594764+0000","last_clean":"2024-09-13T14:48:52.594764+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:48:52.594764+0000","last_undegraded":"2024-09-13T14:48:52.594764+0000","last_fullsized":"2024-09-13T14:48:52.594764+0000","mapping_epoch":326,"log_start":"307'2800","ondisk_log_start":"307'2800","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71434240,"num_objects":1090,"num_object_clones":0,"num_object_copies":3270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10398,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":380,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.363841+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:48:34.363841+0000","last_peered":"2024-09-13T14:48:34.363841+0000","last_clean":"2024-09-13T14:48:34.363841+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:48:34.363841+0000","last_undegraded":"2024-09-13T14:48:34.363841+0000","last_fullsized":"2024-09-13T14:48:34.363841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353558+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:48:34.353558+0000","last_peered":"2024-09-13T14:48:34.353558+0000","last_clean":"2024-09-13T14:48:34.353558+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:48:34.353558+0000","last_undegraded":"2024-09-13T14:48:34.353558+0000","last_fullsized":"2024-09-13T14:48:34.353558+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"369'10695","reported_seq":10842,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:53.899100+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:48:53.899100+0000","last_peered":"2024-09-13T14:48:53.899100+0000","last_clean":"2024-09-13T14:48:53.899100+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:48:53.899100+0000","last_undegraded":"2024-09-13T14:48:53.899100+0000","last_fullsized":"2024-09-13T14:48:53.899100+0000","mapping_epoch":303,"log_start":"307'2100","ondisk_log_start":"307'2100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8595,"log_dups_size":0,"ondisk_log_size":8595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10695,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":334,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.365611+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:48:34.365611+0000","last_peered":"2024-09-13T14:48:34.365611+0000","last_clean":"2024-09-13T14:48:34.365611+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:48:34.365611+0000","last_undegraded":"2024-09-13T14:48:34.365611+0000","last_fullsized":"2024-09-13T14:48:34.365611+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":349,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346312+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:48:34.346312+0000","last_peered":"2024-09-13T14:48:34.346312+0000","last_clean":"2024-09-13T14:48:34.346312+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:48:34.346312+0000","last_undegraded":"2024-09-13T14:48:34.346312+0000","last_fullsized":"2024-09-13T14:48:34.346312+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"369'10507","reported_seq":10654,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:55.022881+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:48:55.022881+0000","last_peered":"2024-09-13T14:48:55.022881+0000","last_clean":"2024-09-13T14:48:55.022881+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:48:55.022881+0000","last_undegraded":"2024-09-13T14:48:55.022881+0000","last_fullsized":"2024-09-13T14:48:55.022881+0000","mapping_epoch":303,"log_start":"307'3000","ondisk_log_start":"307'3000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7507,"log_dups_size":0,"ondisk_log_size":7507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091904,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10507,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":297,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348553+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:48:34.348553+0000","last_peered":"2024-09-13T14:48:34.348553+0000","last_clean":"2024-09-13T14:48:34.348553+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:48:34.348553+0000","last_undegraded":"2024-09-13T14:48:34.348553+0000","last_fullsized":"2024-09-13T14:48:34.348553+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":635,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414680+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:48:50.414680+0000","last_peered":"2024-09-13T14:48:50.414680+0000","last_clean":"2024-09-13T14:48:50.414680+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:48:50.414680+0000","last_undegraded":"2024-09-13T14:48:50.414680+0000","last_fullsized":"2024-09-13T14:48:50.414680+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"369'10437","reported_seq":10581,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:55.773117+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:48:55.773117+0000","last_peered":"2024-09-13T14:48:55.773117+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:48:55.773117+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"308'3400","ondisk_log_start":"308'3400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67174400,"num_objects":1025,"num_object_clones":0,"num_object_copies":3075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10437,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-4608}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":480,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.362801+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:48:34.362801+0000","last_peered":"2024-09-13T14:48:34.362801+0000","last_clean":"2024-09-13T14:48:34.362801+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:48:34.362801+0000","last_undegraded":"2024-09-13T14:48:34.362801+0000","last_fullsized":"2024-09-13T14:48:34.362801+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":633,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402010+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:48:50.402010+0000","last_peered":"2024-09-13T14:48:50.402010+0000","last_clean":"2024-09-13T14:48:50.402010+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:48:50.402010+0000","last_undegraded":"2024-09-13T14:48:50.402010+0000","last_fullsized":"2024-09-13T14:48:50.402010+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353675+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:48:34.353675+0000","last_peered":"2024-09-13T14:48:34.353675+0000","last_clean":"2024-09-13T14:48:34.353675+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:48:34.353675+0000","last_undegraded":"2024-09-13T14:48:34.353675+0000","last_fullsized":"2024-09-13T14:48:34.353675+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"369'10082","reported_seq":10229,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:55.013171+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:48:55.013171+0000","last_peered":"2024-09-13T14:48:55.013171+0000","last_clean":"2024-09-13T14:48:55.013171+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:48:55.013171+0000","last_undegraded":"2024-09-13T14:48:55.013171+0000","last_fullsized":"2024-09-13T14:48:55.013171+0000","mapping_epoch":303,"log_start":"307'2500","ondisk_log_start":"307'2500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322432,"num_objects":1012,"num_object_clones":0,"num_object_copies":3036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10082,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":184,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.351119+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:48:34.351119+0000","last_peered":"2024-09-13T14:48:34.351119+0000","last_clean":"2024-09-13T14:48:34.351119+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:48:34.351119+0000","last_undegraded":"2024-09-13T14:48:34.351119+0000","last_fullsized":"2024-09-13T14:48:34.351119+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":370,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346195+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:48:34.346195+0000","last_peered":"2024-09-13T14:48:34.346195+0000","last_clean":"2024-09-13T14:48:34.346195+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:48:34.346195+0000","last_undegraded":"2024-09-13T14:48:34.346195+0000","last_fullsized":"2024-09-13T14:48:34.346195+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":82,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353993+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:48:34.353993+0000","last_peered":"2024-09-13T14:48:34.353993+0000","last_clean":"2024-09-13T14:48:34.353993+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:48:34.353993+0000","last_undegraded":"2024-09-13T14:48:34.353993+0000","last_fullsized":"2024-09-13T14:48:34.353993+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"369'10265","reported_seq":23675,"reported_epoch":369,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:52.723832+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:48:52.723832+0000","last_peered":"2024-09-13T14:48:52.723832+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:48:52.723832+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"307'2900","ondisk_log_start":"307'2900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75169792,"num_objects":1147,"num_object_clones":0,"num_object_copies":3441,"num_objects_missing_on_primary":90,"num_objects_missing":90,"num_objects_degraded":465,"num_objects_misplaced":1043,"num_objects_unfound":0,"num_objects_dirty":1147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10265,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4560,"num_bytes_recovered":298844160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":90},{"shards":"0,3,4","objects":953},{"shards":"0,4","objects":-271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353400+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:48:34.353400+0000","last_peered":"2024-09-13T14:48:34.353400+0000","last_clean":"2024-09-13T14:48:34.353400+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:48:34.353400+0000","last_undegraded":"2024-09-13T14:48:34.353400+0000","last_fullsized":"2024-09-13T14:48:34.353400+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353871+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:48:34.353871+0000","last_peered":"2024-09-13T14:48:34.353871+0000","last_clean":"2024-09-13T14:48:34.353871+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:48:34.353871+0000","last_undegraded":"2024-09-13T14:48:34.353871+0000","last_fullsized":"2024-09-13T14:48:34.353871+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"369'10103","reported_seq":10250,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:54.959482+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:48:54.959482+0000","last_peered":"2024-09-13T14:48:54.959482+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:48:54.959482+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"307'2600","ondisk_log_start":"307'2600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10103,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-4411}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":177,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.351172+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:48:34.351172+0000","last_peered":"2024-09-13T14:48:34.351172+0000","last_clean":"2024-09-13T14:48:34.351172+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:48:34.351172+0000","last_undegraded":"2024-09-13T14:48:34.351172+0000","last_fullsized":"2024-09-13T14:48:34.351172+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":625,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.407251+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:48:50.407251+0000","last_peered":"2024-09-13T14:48:50.407251+0000","last_clean":"2024-09-13T14:48:50.407251+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:48:50.407251+0000","last_undegraded":"2024-09-13T14:48:50.407251+0000","last_fullsized":"2024-09-13T14:48:50.407251+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":587,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.404105+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:48:50.404105+0000","last_peered":"2024-09-13T14:48:50.404105+0000","last_clean":"2024-09-13T14:48:50.404105+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:48:50.404105+0000","last_undegraded":"2024-09-13T14:48:50.404105+0000","last_fullsized":"2024-09-13T14:48:50.404105+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"369'10120","reported_seq":10201,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:52.543122+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:48:52.543122+0000","last_peered":"2024-09-13T14:48:52.543122+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:48:52.543122+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"307'2600","ondisk_log_start":"307'2600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7520,"log_dups_size":0,"ondisk_log_size":7520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72089600,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10120,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-4396}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":471,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.363503+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:48:34.363503+0000","last_peered":"2024-09-13T14:48:34.363503+0000","last_clean":"2024-09-13T14:48:34.363503+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:48:34.363503+0000","last_undegraded":"2024-09-13T14:48:34.363503+0000","last_fullsized":"2024-09-13T14:48:34.363503+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":893,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348582+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:48:34.348582+0000","last_peered":"2024-09-13T14:48:34.348582+0000","last_clean":"2024-09-13T14:48:34.348582+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:48:34.348582+0000","last_undegraded":"2024-09-13T14:48:34.348582+0000","last_fullsized":"2024-09-13T14:48:34.348582+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":578,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400703+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:48:50.400703+0000","last_peered":"2024-09-13T14:48:50.400703+0000","last_clean":"2024-09-13T14:48:50.400703+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:48:50.400703+0000","last_undegraded":"2024-09-13T14:48:50.400703+0000","last_fullsized":"2024-09-13T14:48:50.400703+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":335,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346662+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:48:34.346662+0000","last_peered":"2024-09-13T14:48:34.346662+0000","last_clean":"2024-09-13T14:48:34.346662+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:48:34.346662+0000","last_undegraded":"2024-09-13T14:48:34.346662+0000","last_fullsized":"2024-09-13T14:48:34.346662+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"369'10459","reported_seq":10604,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:54.983852+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:48:54.983852+0000","last_peered":"2024-09-13T14:48:54.983852+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:48:54.983852+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"307'2900","ondisk_log_start":"307'2900","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7559,"log_dups_size":0,"ondisk_log_size":7559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71368704,"num_objects":1089,"num_object_clones":0,"num_object_copies":3267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10459,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-4591}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":432,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.408547+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:48:50.408547+0000","last_peered":"2024-09-13T14:48:50.408547+0000","last_clean":"2024-09-13T14:48:50.408547+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:48:50.408547+0000","last_undegraded":"2024-09-13T14:48:50.408547+0000","last_fullsized":"2024-09-13T14:48:50.408547+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":759,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.362999+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:48:34.362999+0000","last_peered":"2024-09-13T14:48:34.362999+0000","last_clean":"2024-09-13T14:48:34.362999+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:48:34.362999+0000","last_undegraded":"2024-09-13T14:48:34.362999+0000","last_fullsized":"2024-09-13T14:48:34.362999+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":628,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.417020+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:48:50.417020+0000","last_peered":"2024-09-13T14:48:50.417020+0000","last_clean":"2024-09-13T14:48:50.417020+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:48:50.417020+0000","last_undegraded":"2024-09-13T14:48:50.417020+0000","last_fullsized":"2024-09-13T14:48:50.417020+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":632,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.415214+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:48:50.415214+0000","last_peered":"2024-09-13T14:48:50.415214+0000","last_clean":"2024-09-13T14:48:50.415214+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:48:50.415214+0000","last_undegraded":"2024-09-13T14:48:50.415214+0000","last_fullsized":"2024-09-13T14:48:50.415214+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"369'10576","reported_seq":20264,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:55.708117+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:48:55.708117+0000","last_peered":"2024-09-13T14:48:55.708117+0000","last_clean":"2024-09-13T14:48:55.708117+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:48:55.708117+0000","last_undegraded":"2024-09-13T14:48:55.708117+0000","last_fullsized":"2024-09-13T14:48:55.708117+0000","mapping_epoch":330,"log_start":"308'3500","ondisk_log_start":"308'3500","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10576,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":500,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.362913+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:48:34.362913+0000","last_peered":"2024-09-13T14:48:34.362913+0000","last_clean":"2024-09-13T14:48:34.362913+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:48:34.362913+0000","last_undegraded":"2024-09-13T14:48:34.362913+0000","last_fullsized":"2024-09-13T14:48:34.362913+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":596,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400528+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:48:50.400528+0000","last_peered":"2024-09-13T14:48:50.400528+0000","last_clean":"2024-09-13T14:48:50.400528+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:48:50.400528+0000","last_undegraded":"2024-09-13T14:48:50.400528+0000","last_fullsized":"2024-09-13T14:48:50.400528+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":565,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403766+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:48:50.403766+0000","last_peered":"2024-09-13T14:48:50.403766+0000","last_clean":"2024-09-13T14:48:50.403766+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:48:50.403766+0000","last_undegraded":"2024-09-13T14:48:50.403766+0000","last_fullsized":"2024-09-13T14:48:50.403766+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":118,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348914+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:48:34.348914+0000","last_peered":"2024-09-13T14:48:34.348914+0000","last_clean":"2024-09-13T14:48:34.348914+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:48:34.348914+0000","last_undegraded":"2024-09-13T14:48:34.348914+0000","last_fullsized":"2024-09-13T14:48:34.348914+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"369'10208","reported_seq":10354,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:48:55.721191+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:48:55.721191+0000","last_peered":"2024-09-13T14:48:55.721191+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:48:55.721191+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"308'3200","ondisk_log_start":"308'3200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10208,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-4487}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":177,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348490+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:48:34.348490+0000","last_peered":"2024-09-13T14:48:34.348490+0000","last_clean":"2024-09-13T14:48:34.348490+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:48:34.348490+0000","last_undegraded":"2024-09-13T14:48:34.348490+0000","last_fullsized":"2024-09-13T14:48:34.348490+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":984,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.351222+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:48:34.351222+0000","last_peered":"2024-09-13T14:48:34.351222+0000","last_clean":"2024-09-13T14:48:34.351222+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:48:34.351222+0000","last_undegraded":"2024-09-13T14:48:34.351222+0000","last_fullsized":"2024-09-13T14:48:34.351222+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353920+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:48:34.353920+0000","last_peered":"2024-09-13T14:48:34.353920+0000","last_clean":"2024-09-13T14:48:34.353920+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:48:34.353920+0000","last_undegraded":"2024-09-13T14:48:34.353920+0000","last_fullsized":"2024-09-13T14:48:34.353920+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":585,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403690+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:48:50.403690+0000","last_peered":"2024-09-13T14:48:50.403690+0000","last_clean":"2024-09-13T14:48:50.403690+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:48:50.403690+0000","last_undegraded":"2024-09-13T14:48:50.403690+0000","last_fullsized":"2024-09-13T14:48:50.403690+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"369'10257","reported_seq":19684,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:53.991307+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:48:53.991307+0000","last_peered":"2024-09-13T14:48:53.991307+0000","last_clean":"2024-09-13T14:48:53.991307+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:48:53.991307+0000","last_undegraded":"2024-09-13T14:48:53.991307+0000","last_fullsized":"2024-09-13T14:48:53.991307+0000","mapping_epoch":337,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8657,"log_dups_size":0,"ondisk_log_size":8657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10257,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":368,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346809+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:48:34.346809+0000","last_peered":"2024-09-13T14:48:34.346809+0000","last_clean":"2024-09-13T14:48:34.346809+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:48:34.346809+0000","last_undegraded":"2024-09-13T14:48:34.346809+0000","last_fullsized":"2024-09-13T14:48:34.346809+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348739+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:48:34.348739+0000","last_peered":"2024-09-13T14:48:34.348739+0000","last_clean":"2024-09-13T14:48:34.348739+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:48:34.348739+0000","last_undegraded":"2024-09-13T14:48:34.348739+0000","last_fullsized":"2024-09-13T14:48:34.348739+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1164,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403875+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:48:50.403875+0000","last_peered":"2024-09-13T14:48:50.403875+0000","last_clean":"2024-09-13T14:48:50.403875+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:48:50.403875+0000","last_undegraded":"2024-09-13T14:48:50.403875+0000","last_fullsized":"2024-09-13T14:48:50.403875+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":566,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.405531+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:48:50.405531+0000","last_peered":"2024-09-13T14:48:50.405531+0000","last_clean":"2024-09-13T14:48:50.405531+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:48:50.405531+0000","last_undegraded":"2024-09-13T14:48:50.405531+0000","last_fullsized":"2024-09-13T14:48:50.405531+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"369'10171","reported_seq":10318,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:55.802910+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:48:55.802910+0000","last_peered":"2024-09-13T14:48:55.802910+0000","last_clean":"2024-09-13T14:48:55.802910+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:48:55.802910+0000","last_undegraded":"2024-09-13T14:48:55.802910+0000","last_fullsized":"2024-09-13T14:48:55.802910+0000","mapping_epoch":303,"log_start":"307'3100","ondisk_log_start":"307'3100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67698688,"num_objects":1033,"num_object_clones":0,"num_object_copies":3099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10171,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":368,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346697+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:48:34.346697+0000","last_peered":"2024-09-13T14:48:34.346697+0000","last_clean":"2024-09-13T14:48:34.346697+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:48:34.346697+0000","last_undegraded":"2024-09-13T14:48:34.346697+0000","last_fullsized":"2024-09-13T14:48:34.346697+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":575,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403786+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:48:50.403786+0000","last_peered":"2024-09-13T14:48:50.403786+0000","last_clean":"2024-09-13T14:48:50.403786+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:48:50.403786+0000","last_undegraded":"2024-09-13T14:48:50.403786+0000","last_fullsized":"2024-09-13T14:48:50.403786+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":763,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.365668+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:48:34.365668+0000","last_peered":"2024-09-13T14:48:34.365668+0000","last_clean":"2024-09-13T14:48:34.365668+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:48:34.365668+0000","last_undegraded":"2024-09-13T14:48:34.365668+0000","last_fullsized":"2024-09-13T14:48:34.365668+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":611,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414581+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:48:50.414581+0000","last_peered":"2024-09-13T14:48:50.414581+0000","last_clean":"2024-09-13T14:48:50.414581+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:48:50.414581+0000","last_undegraded":"2024-09-13T14:48:50.414581+0000","last_fullsized":"2024-09-13T14:48:50.414581+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"369'10603","reported_seq":10750,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:53.993388+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:48:53.993388+0000","last_peered":"2024-09-13T14:48:53.993388+0000","last_clean":"2024-09-13T14:48:53.993388+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:48:53.993388+0000","last_undegraded":"2024-09-13T14:48:53.993388+0000","last_fullsized":"2024-09-13T14:48:53.993388+0000","mapping_epoch":303,"log_start":"307'2000","ondisk_log_start":"307'2000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8603,"log_dups_size":0,"ondisk_log_size":8603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71237632,"num_objects":1087,"num_object_clones":0,"num_object_copies":3261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10603,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353832+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:48:34.353832+0000","last_peered":"2024-09-13T14:48:34.353832+0000","last_clean":"2024-09-13T14:48:34.353832+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:48:34.353832+0000","last_undegraded":"2024-09-13T14:48:34.353832+0000","last_fullsized":"2024-09-13T14:48:34.353832+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":611,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402842+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:48:50.402842+0000","last_peered":"2024-09-13T14:48:50.402842+0000","last_clean":"2024-09-13T14:48:50.402842+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:48:50.402842+0000","last_undegraded":"2024-09-13T14:48:50.402842+0000","last_fullsized":"2024-09-13T14:48:50.402842+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":951,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.404424+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:48:50.404424+0000","last_peered":"2024-09-13T14:48:50.404424+0000","last_clean":"2024-09-13T14:48:50.404424+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:48:50.404424+0000","last_undegraded":"2024-09-13T14:48:50.404424+0000","last_fullsized":"2024-09-13T14:48:50.404424+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":609,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.407082+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:48:50.407082+0000","last_peered":"2024-09-13T14:48:50.407082+0000","last_clean":"2024-09-13T14:48:50.407082+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:48:50.407082+0000","last_undegraded":"2024-09-13T14:48:50.407082+0000","last_fullsized":"2024-09-13T14:48:50.407082+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"369'10442","reported_seq":21822,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:52.470406+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:48:52.470406+0000","last_peered":"2024-09-13T14:48:52.470406+0000","last_clean":"2024-09-13T14:48:52.470406+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:48:52.470406+0000","last_undegraded":"2024-09-13T14:48:52.470406+0000","last_fullsized":"2024-09-13T14:48:52.470406+0000","mapping_epoch":332,"log_start":"306'1300","ondisk_log_start":"306'1300","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9142,"log_dups_size":0,"ondisk_log_size":9142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70189080,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10442,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":462,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402137+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:48:50.402137+0000","last_peered":"2024-09-13T14:48:50.402137+0000","last_clean":"2024-09-13T14:48:50.402137+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:48:50.402137+0000","last_undegraded":"2024-09-13T14:48:50.402137+0000","last_fullsized":"2024-09-13T14:48:50.402137+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":646,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414971+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:48:50.414971+0000","last_peered":"2024-09-13T14:48:50.414971+0000","last_clean":"2024-09-13T14:48:50.414971+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:48:50.414971+0000","last_undegraded":"2024-09-13T14:48:50.414971+0000","last_fullsized":"2024-09-13T14:48:50.414971+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":787,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346915+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:48:34.346915+0000","last_peered":"2024-09-13T14:48:34.346915+0000","last_clean":"2024-09-13T14:48:34.346915+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:48:34.346915+0000","last_undegraded":"2024-09-13T14:48:34.346915+0000","last_fullsized":"2024-09-13T14:48:34.346915+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353495+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:48:34.353495+0000","last_peered":"2024-09-13T14:48:34.353495+0000","last_clean":"2024-09-13T14:48:34.353495+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:48:34.353495+0000","last_undegraded":"2024-09-13T14:48:34.353495+0000","last_fullsized":"2024-09-13T14:48:34.353495+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353925+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:48:34.353925+0000","last_peered":"2024-09-13T14:48:34.353925+0000","last_clean":"2024-09-13T14:48:34.353925+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:48:34.353925+0000","last_undegraded":"2024-09-13T14:48:34.353925+0000","last_fullsized":"2024-09-13T14:48:34.353925+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":78,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353595+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:48:34.353595+0000","last_peered":"2024-09-13T14:48:34.353595+0000","last_clean":"2024-09-13T14:48:34.353595+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:48:34.353595+0000","last_undegraded":"2024-09-13T14:48:34.353595+0000","last_fullsized":"2024-09-13T14:48:34.353595+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":563,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.405388+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:48:50.405388+0000","last_peered":"2024-09-13T14:48:50.405388+0000","last_clean":"2024-09-13T14:48:50.405388+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:48:50.405388+0000","last_undegraded":"2024-09-13T14:48:50.405388+0000","last_fullsized":"2024-09-13T14:48:50.405388+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":368,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.365598+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:48:34.365598+0000","last_peered":"2024-09-13T14:48:34.365598+0000","last_clean":"2024-09-13T14:48:34.365598+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:48:34.365598+0000","last_undegraded":"2024-09-13T14:48:34.365598+0000","last_fullsized":"2024-09-13T14:48:34.365598+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":334,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348826+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:48:34.348826+0000","last_peered":"2024-09-13T14:48:34.348826+0000","last_clean":"2024-09-13T14:48:34.348826+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:48:34.348826+0000","last_undegraded":"2024-09-13T14:48:34.348826+0000","last_fullsized":"2024-09-13T14:48:34.348826+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":567,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400267+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:48:50.400267+0000","last_peered":"2024-09-13T14:48:50.400267+0000","last_clean":"2024-09-13T14:48:50.400267+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:48:50.400267+0000","last_undegraded":"2024-09-13T14:48:50.400267+0000","last_fullsized":"2024-09-13T14:48:50.400267+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":619,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414881+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:48:50.414881+0000","last_peered":"2024-09-13T14:48:50.414881+0000","last_clean":"2024-09-13T14:48:50.414881+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:48:50.414881+0000","last_undegraded":"2024-09-13T14:48:50.414881+0000","last_fullsized":"2024-09-13T14:48:50.414881+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":623,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.407000+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:48:50.407000+0000","last_peered":"2024-09-13T14:48:50.407000+0000","last_clean":"2024-09-13T14:48:50.407000+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:48:50.407000+0000","last_undegraded":"2024-09-13T14:48:50.407000+0000","last_fullsized":"2024-09-13T14:48:50.407000+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":366,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.346863+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:48:34.346863+0000","last_peered":"2024-09-13T14:48:34.346863+0000","last_clean":"2024-09-13T14:48:34.346863+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:48:34.346863+0000","last_undegraded":"2024-09-13T14:48:34.346863+0000","last_fullsized":"2024-09-13T14:48:34.346863+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":641,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.407161+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:48:50.407161+0000","last_peered":"2024-09-13T14:48:50.407161+0000","last_clean":"2024-09-13T14:48:50.407161+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:48:50.407161+0000","last_undegraded":"2024-09-13T14:48:50.407161+0000","last_fullsized":"2024-09-13T14:48:50.407161+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":611,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.415141+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:48:50.415141+0000","last_peered":"2024-09-13T14:48:50.415141+0000","last_clean":"2024-09-13T14:48:50.415141+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:48:50.415141+0000","last_undegraded":"2024-09-13T14:48:50.415141+0000","last_fullsized":"2024-09-13T14:48:50.415141+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":587,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.404386+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:48:50.404386+0000","last_peered":"2024-09-13T14:48:50.404386+0000","last_clean":"2024-09-13T14:48:50.404386+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:48:50.404386+0000","last_undegraded":"2024-09-13T14:48:50.404386+0000","last_fullsized":"2024-09-13T14:48:50.404386+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":636,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402303+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:48:50.402303+0000","last_peered":"2024-09-13T14:48:50.402303+0000","last_clean":"2024-09-13T14:48:50.402303+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:48:50.402303+0000","last_undegraded":"2024-09-13T14:48:50.402303+0000","last_fullsized":"2024-09-13T14:48:50.402303+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.385089+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:48:34.385089+0000","last_peered":"2024-09-13T14:48:34.385089+0000","last_clean":"2024-09-13T14:48:34.385089+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:48:34.385089+0000","last_undegraded":"2024-09-13T14:48:34.385089+0000","last_fullsized":"2024-09-13T14:48:34.385089+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":533,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403986+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:48:50.403986+0000","last_peered":"2024-09-13T14:48:50.403986+0000","last_clean":"2024-09-13T14:48:50.403986+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:48:50.403986+0000","last_undegraded":"2024-09-13T14:48:50.403986+0000","last_fullsized":"2024-09-13T14:48:50.403986+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":614,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414577+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:48:50.414577+0000","last_peered":"2024-09-13T14:48:50.414577+0000","last_clean":"2024-09-13T14:48:50.414577+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:48:50.414577+0000","last_undegraded":"2024-09-13T14:48:50.414577+0000","last_fullsized":"2024-09-13T14:48:50.414577+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353832+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:48:34.353832+0000","last_peered":"2024-09-13T14:48:34.353832+0000","last_clean":"2024-09-13T14:48:34.353832+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:48:34.353832+0000","last_undegraded":"2024-09-13T14:48:34.353832+0000","last_fullsized":"2024-09-13T14:48:34.353832+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":334,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348629+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:48:34.348629+0000","last_peered":"2024-09-13T14:48:34.348629+0000","last_clean":"2024-09-13T14:48:34.348629+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:48:34.348629+0000","last_undegraded":"2024-09-13T14:48:34.348629+0000","last_fullsized":"2024-09-13T14:48:34.348629+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.385002+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:48:34.385002+0000","last_peered":"2024-09-13T14:48:34.385002+0000","last_clean":"2024-09-13T14:48:34.385002+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:48:34.385002+0000","last_undegraded":"2024-09-13T14:48:34.385002+0000","last_fullsized":"2024-09-13T14:48:34.385002+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":300,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.348942+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:48:34.348942+0000","last_peered":"2024-09-13T14:48:34.348942+0000","last_clean":"2024-09-13T14:48:34.348942+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:48:34.348942+0000","last_undegraded":"2024-09-13T14:48:34.348942+0000","last_fullsized":"2024-09-13T14:48:34.348942+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":633,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400623+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:48:50.400623+0000","last_peered":"2024-09-13T14:48:50.400623+0000","last_clean":"2024-09-13T14:48:50.400623+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:48:50.400623+0000","last_undegraded":"2024-09-13T14:48:50.400623+0000","last_fullsized":"2024-09-13T14:48:50.400623+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":620,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402365+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:48:50.402365+0000","last_peered":"2024-09-13T14:48:50.402365+0000","last_clean":"2024-09-13T14:48:50.402365+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:48:50.402365+0000","last_undegraded":"2024-09-13T14:48:50.402365+0000","last_fullsized":"2024-09-13T14:48:50.402365+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T14:48:34.353371+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:48:34.353371+0000","last_peered":"2024-09-13T14:48:34.353371+0000","last_clean":"2024-09-13T14:48:34.353371+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:48:34.353371+0000","last_undegraded":"2024-09-13T14:48:34.353371+0000","last_fullsized":"2024-09-13T14:48:34.353371+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":625,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.414869+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:48:50.414869+0000","last_peered":"2024-09-13T14:48:50.414869+0000","last_clean":"2024-09-13T14:48:50.414869+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:48:50.414869+0000","last_undegraded":"2024-09-13T14:48:50.414869+0000","last_fullsized":"2024-09-13T14:48:50.414869+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1115750424,"num_objects":17026,"num_object_clones":0,"num_object_copies":51078,"num_objects_missing_on_primary":90,"num_objects_missing":90,"num_objects_degraded":29132,"num_objects_misplaced":1043,"num_objects_unfound":0,"num_objects_dirty":17026,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165652,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29018,"num_bytes_recovered":1841299456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3947704320,"data_stored":3947692104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125552,"ondisk_log_size":125552,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397953,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263760,"kb_used_data":363588,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":92481392,"statfs":{"total":95995035648,"available":94700945408,"internally_reserved":0,"allocated":372314112,"data_stored":370325610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594181,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1612472,"kb_used_data":713840,"kb_used_omap":1,"kb_used_meta":898622,"kb_avail":92132680,"statfs":{"total":95995035648,"available":94343864320,"internally_reserved":0,"allocated":730972160,"data_stored":729003412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":920189272},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665635,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":457336,"kb_used_data":120624,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":93287816,"statfs":{"total":95995035648,"available":95526723584,"internally_reserved":0,"allocated":123518976,"data_stored":121546559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986637,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259656,"kb_used_data":355644,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":92485496,"statfs":{"total":95995035648,"available":94705147904,"internally_reserved":0,"allocated":364179456,"data_stored":362199052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697197,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1517300,"kb_used_data":767468,"kb_used_omap":4,"kb_used_meta":749819,"kb_avail":92227852,"statfs":{"total":95995035648,"available":94441320448,"internally_reserved":0,"allocated":785887232,"data_stored":783918968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":767814731},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860724,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333244,"kb_used_data":702388,"kb_used_omap":8,"kb_used_meta":630839,"kb_avail":92411908,"statfs":{"total":95995035648,"available":94629793792,"internally_reserved":0,"allocated":719245312,"data_stored":717269394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":645979980},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020395,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796912,"kb_used_data":483624,"kb_used_omap":9,"kb_used_meta":313270,"kb_avail":92948240,"statfs":{"total":95995035648,"available":95178997760,"internally_reserved":0,"allocated":495230976,"data_stored":493266185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":320789317},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187785,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064112,"kb_used_data":427816,"kb_used_omap":7,"kb_used_meta":636216,"kb_avail":92681040,"statfs":{"total":95995035648,"available":94905384960,"internally_reserved":0,"allocated":438083584,"data_stored":436119059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":651485515},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":425590784,"data_stored":425590784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":484507648,"data_stored":484507648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":703598592,"data_stored":703594520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":777125888,"data_stored":777125888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":354877440,"data_stored":354877440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":118034432,"data_stored":118030360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":723976192,"data_stored":723976192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":359993344,"data_stored":359989272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:49:00.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:49:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:00.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.349 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:00.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:00.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:00.491 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:00.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:00.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:00.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:00 smithi017 ceph-mon[103001]: from='client.46619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:00.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:00 smithi017 ceph-mon[103001]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1008 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 29044/48243 objects degraded (60.204%); 890/48243 objects misplaced (1.845%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:00 smithi017 ceph-mon[107556]: from='client.46619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:00.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:00 smithi017 ceph-mon[107556]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1008 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 29044/48243 objects degraded (60.204%); 890/48243 objects misplaced (1.845%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.659 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.670 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:00 smithi189 ceph-mon[90992]: from='client.46619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:00.671 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:00 smithi189 ceph-mon[90992]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1008 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 29044/48243 objects degraded (60.204%); 890/48243 objects misplaced (1.845%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:49:00.687 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:00.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:49:00.716 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:00.742 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:49:00.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:00.769 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:49:00.769 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:00.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:49:00.804 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:00.833 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:49:00.833 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:00.833 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89683 8.8 8.e68ed018 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 497.10793055699997, 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.98019932300000001, 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:49:00.834 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89683 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719758+0000", 2024-09-13T14:49:00.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719757+0000", 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719930+0000", 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017338299999999999 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719942+0000", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1644999999999999e-05 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.719950+0000", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.622e-06 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.720009+0000", 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9120000000000003e-05 2024-09-13T14:49:00.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.720018+0000", 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7010000000000002e-06 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.699946+0000", 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.97992809800000003 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.699958+0000", 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1919000000000001e-05 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89695 8.8 8.d544e9a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 497.089738893, 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96264634999999998, 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.839 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89695 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737950+0000", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.737947+0000", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738028+0000", 2024-09-13T14:49:00.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0275e-05 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738048+0000", 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0489999999999999e-05 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738056+0000", 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1519999999999992e-06 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738110+0000", 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3384999999999998e-05 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.738119+0000", 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9080000000000001e-06 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.700588+0000", 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.96246986300000004 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.700596+0000", 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8140000000000001e-06 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89697 8.d 8.41b848dd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 497.08587572099998, 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95496279900000003, 2024-09-13T14:49:00.844 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89697 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741813+0000", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741811+0000", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741897+0000", 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6118000000000005e-05 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741917+0000", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0013999999999998e-05 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741924+0000", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.525e-06 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741979+0000", 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4194000000000003e-05 2024-09-13T14:49:00.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.741986+0000", 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7270000000000007e-06 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696767+0000", 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95478041400000002 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696776+0000", 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1290000000000002e-06 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:89699 8.d 8.b203ad2d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 497.08206444899997, 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95118529100000004, 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:49:00.849 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 89699 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745624+0000", 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745623+0000", 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745687+0000", 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4147999999999996e-05 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745703+0000", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5985e-05 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745711+0000", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2900000000000002e-06 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745763+0000", 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1915e-05 2024-09-13T14:49:00.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:43.745770+0000", 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.17e-06 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696804+0000", 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95103346799999999 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:40:44.696810+0000", 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8499999999999999e-06 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:127572 8.d 8:b5efe93e:::benchmark_data_smithi017_253885_object36231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:45:55.302540+0000", 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 185.52514852900001, 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087325372999999998, 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:49:00.854 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 127572 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302540+0000", 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302540+0000", 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302539+0000", 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302545+0000", 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.251e-06 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302557+0000", 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2104e-05 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302562+0000", 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6650000000000002e-06 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302608+0000", 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6007000000000002e-05 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302729+0000", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000121426 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.302844+0000", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011492700000000001 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.305702+0000", 2024-09-13T14:49:00.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028574270000000001 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.306558+0000", 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00085630899999999998 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.389797+0000", 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.083238902000000004 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.389838+0000", 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1298999999999999e-05 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:45:55.389866+0000", 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7294e-05 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:167490 8.0 e369/317 8:085e1852:::benchmark_data_smithi017_253885_object76149:head v 369'10535, mlcod=0'0)", 2024-09-13T14:49:00.860 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.271167+0000", 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.5565214960000002, 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.081881941, 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271167+0000", 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271167+0000", 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271167+0000", 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271171+0000", 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9700000000000001e-06 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271179+0000", 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9410000000000007e-06 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.271182+0000", 2024-09-13T14:49:00.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:49:00.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5039999999999998e-06 2024-09-13T14:49:00.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.287383+0000", 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016200645999999999 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.287409+0000", 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5959999999999999e-05 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.353001+0000", 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.065591978999999995 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.353049+0000", 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8210000000000001e-05 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.866 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:167589 8.9 e369/317 8:9382d3fd:::benchmark_data_smithi017_253885_object76248:head v 369'10312, mlcod=0'0)", 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.429657+0000", 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.3980318609999998, 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.069257072000000003, 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429657+0000", 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429657+0000", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429656+0000", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429665+0000", 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9360000000000005e-06 2024-09-13T14:49:00.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429678+0000", 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3176999999999999e-05 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429685+0000", 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6630000000000004e-06 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.445145+0000", 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015460701 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.445173+0000", 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7729000000000002e-05 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.498614+0000", 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.053440975000000002 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.498914+0000", 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00029998299999999998 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:167670 8.0 e369/317 8:03636769:::benchmark_data_smithi017_253885_object76329:head v 369'10545, mlcod=0'0)", 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.535151+0000", 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.2925380999999998, 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084166464999999996, 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.871 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535151+0000", 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535151+0000", 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535149+0000", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535162+0000", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3079e-05 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535183+0000", 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1611999999999999e-05 2024-09-13T14:49:00.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.535193+0000", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2779999999999999e-06 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.552457+0000", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017264638999999998 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.552478+0000", 2024-09-13T14:49:00.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1279000000000001e-05 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619271+0000", 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.066792510999999999 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619317+0000", 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6144999999999999e-05 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:167671 8.0 e369/317 8:0092cff1:::benchmark_data_smithi017_253885_object76330:head v 369'10546, mlcod=0'0)", 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.536738+0000", 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.2909505640000001, 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.082654232999999994, 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536738+0000", 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536738+0000", 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536737+0000", 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536750+0000", 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3569e-05 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536771+0000", 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0956999999999999e-05 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.536780+0000", 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3880000000000006e-06 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.569850+0000", 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.033069121999999999 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.569877+0000", 2024-09-13T14:49:00.879 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:00.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7285999999999999e-05 2024-09-13T14:49:00.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619352+0000", 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049474991000000003 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619392+0000", 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0639999999999997e-05 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:167678 8.8 e369/317 8:19edbe09:::benchmark_data_smithi017_253885_object76337:head v 369'10309, mlcod=0'0)", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.548452+0000", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.2792364900000002, 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.070897194999999996, 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548452+0000", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548452+0000", 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548451+0000", 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548458+0000", 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3139999999999998e-06 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548473+0000", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4371e-05 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.548479+0000", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8969999999999996e-06 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.567194+0000", 2024-09-13T14:49:00.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018714992999999999 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.567216+0000", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1858999999999999e-05 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619322+0000", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052106524000000001 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.619349+0000", 2024-09-13T14:49:00.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7348999999999999e-05 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168700 8.8 e369/317 8:16925a37:::benchmark_data_smithi017_253885_object77359:head v 369'10383, mlcod=0'0)", 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:59.896802+0000", 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.93088727100000002, 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075678379000000004, 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896802+0000", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896802+0000", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896801+0000", 2024-09-13T14:49:00.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896805+0000", 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8449999999999996e-06 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896814+0000", 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5809999999999997e-06 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.896817+0000", 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.782e-06 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.922966+0000", 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.026148918 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.923139+0000", 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000172614 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.972388+0000", 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049248834999999998 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.972480+0000", 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2192000000000001e-05 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.891 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168709 8.0 e369/317 8:072df135:::benchmark_data_smithi017_253885_object77368:head v 369'10601, mlcod=0'0)", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:59.910905+0000", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.91678337799999998, 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068661659, 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910905+0000", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910905+0000", 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910904+0000", 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910913+0000", 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7560000000000005e-06 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910925+0000", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2459e-05 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.910932+0000", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9539999999999998e-06 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.940017+0000", 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.029084736999999999 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.940037+0000", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9885e-05 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.979415+0000", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.039377621000000002 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:59.979567+0000", 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015241699999999999 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.895 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168901 8.0 e369/317 8:02c17ce8:::benchmark_data_smithi017_253885_object77560:head v 369'10615, mlcod=0'0)", 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.171935+0000", 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.655753908, 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.076185866000000005, 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171935+0000", 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171935+0000", 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171934+0000", 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171945+0000", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0981e-05 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171956+0000", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0997e-05 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.171960+0000", 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9530000000000001e-06 2024-09-13T14:49:00.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.192408+0000", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020448821999999998 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.192430+0000", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1888000000000001e-05 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.247939+0000", 2024-09-13T14:49:00.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.055508552000000003 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.248121+0000", 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018186400000000001 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168902 8.8 e369/317 8:1c7c470e:::benchmark_data_smithi017_253885_object77561:head v 369'10394, mlcod=0'0)", 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.178030+0000", 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.64965893399999997, 2024-09-13T14:49:00.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.093932062999999996, 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178030+0000", 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178030+0000", 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178029+0000", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178036+0000", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0550000000000002e-06 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178050+0000", 2024-09-13T14:49:00.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4439e-05 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178056+0000", 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6509999999999998e-06 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.192063+0000", 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014007354 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.192091+0000", 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7189999999999999e-05 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.271920+0000", 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079828857000000003 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.271962+0000", 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2410999999999998e-05 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168905 8.0 e369/317 8:05d18295:::benchmark_data_smithi017_253885_object77564:head v 369'10616, mlcod=0'0)", 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.178063+0000", 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.64962602700000005, 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.100364968, 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178063+0000", 2024-09-13T14:49:00.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178063+0000", 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178062+0000", 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178067+0000", 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3080000000000001e-06 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178073+0000", 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9479999999999996e-06 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.178076+0000", 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5710000000000002e-06 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.201936+0000", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023859625999999998 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.201963+0000", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7328e-05 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.278391+0000", 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.076427693000000005 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.278428+0000", 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6789999999999998e-05 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168907 8.8 e369/317 8:148127c7:::benchmark_data_smithi017_253885_object77566:head v 369'10395, mlcod=0'0)", 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.180358+0000", 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.64733122399999998, 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.106020268, 2024-09-13T14:49:00.909 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180358+0000", 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180358+0000", 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180356+0000", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180364+0000", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9780000000000005e-06 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180401+0000", 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6341000000000003e-05 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.180407+0000", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5560000000000004e-06 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.212268+0000", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031861095999999998 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.212291+0000", 2024-09-13T14:49:00.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3093999999999999e-05 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286334+0000", 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.074042254000000002 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286378+0000", 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4328e-05 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168912 8.8 e369/317 8:1e6909f5:::benchmark_data_smithi017_253885_object77571:head v 369'10396, mlcod=0'0)", 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.191748+0000", 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.63594089200000004, 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094662704, 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191748+0000", 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191748+0000", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191747+0000", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191757+0000", 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0232e-05 2024-09-13T14:49:00.915 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191776+0000", 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9026e-05 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.191784+0000", 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7810000000000005e-06 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.221924+0000", 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.030139839000000002 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.221953+0000", 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9567e-05 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286386+0000", 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.064432527000000003 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286411+0000", 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4904000000000001e-05 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168921 8.8 e369/317 8:1d67f55c:::benchmark_data_smithi017_253885_object77580:head v 369'10397, mlcod=0'0)", 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.205839+0000", 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.62184963500000001, 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.080599571999999994, 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.918 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205839+0000", 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205839+0000", 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205838+0000", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205845+0000", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2749999999999998e-06 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205858+0000", 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2294e-05 2024-09-13T14:49:00.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.205864+0000", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3260000000000001e-06 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.230106+0000", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024241848999999999 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.230158+0000", 2024-09-13T14:49:00.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2342999999999997e-05 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286413+0000", 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.056255008000000002 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286439+0000", 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5488000000000001e-05 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.922 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:168928 8.0 e369/317 8:0260c83a:::benchmark_data_smithi017_253885_object77587:head v 369'10617, mlcod=0'0)", 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.217771+0000", 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.60991797199999997, 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.068693665000000001, 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217771+0000", 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217771+0000", 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217770+0000", 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217778+0000", 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.926e-06 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217789+0000", 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.147e-05 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.217795+0000", 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.231135+0000", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01333969 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.231570+0000", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00043530400000000002 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286441+0000", 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.054870365999999997 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.286464+0000", 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3841000000000001e-05 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.35952.0:169288 8.8 e369/317 8:186ff14e:::benchmark_data_smithi017_253885_object77947:head v 369'10425, mlcod=0'0)", 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:00.749666+0000", 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.078023112000000006, 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.071398262000000004, 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:49:00.927 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749666+0000", 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749666+0000", 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749664+0000", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749677+0000", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2368000000000001e-05 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749689+0000", 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2031000000000001e-05 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.749700+0000", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1321999999999999e-05 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.773873+0000", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.024173337999999999 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.773921+0000", 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7355999999999999e-05 2024-09-13T14:49:00.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.821042+0000", 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047121521 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:00.821064+0000", 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1780000000000002e-05 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.931 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:49:00.932 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:49:00.932 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:49:00.932 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:49:00.932 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:00.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:01.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:01.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:01.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:01.552 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2163416572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:01.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:01.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:01.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2163416572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2163416572' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:01.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:01.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:01.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:02.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:02.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:02.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:02.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:02.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:02.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:02.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:49:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:02 smithi189 ceph-mon[90992]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 907 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 29044/43425 objects degraded (66.883%); 890/43425 objects misplaced (2.050%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:02.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:02.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:02.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:02 smithi017 ceph-mon[103001]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 907 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 29044/43425 objects degraded (66.883%); 890/43425 objects misplaced (2.050%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:02.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:02 smithi017 ceph-mon[107556]: pgmap v1215: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 907 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 29044/43425 objects degraded (66.883%); 890/43425 objects misplaced (2.050%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:02.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:02.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.863 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:02.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:02.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:03.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:03.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:03.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:03.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:03.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:03.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:03.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:03.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:03.728 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:03.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:03.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:03.903 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:03.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:03.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:04.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:04.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:04.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:04.338 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:04.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.515 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:04 smithi017 ceph-mon[103001]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 29054/40107 objects degraded (72.441%); 662/40107 objects misplaced (1.651%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:49:04.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29054/40107 objects degraded (72.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:04.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:04 smithi017 ceph-mon[107556]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 29054/40107 objects degraded (72.441%); 662/40107 objects misplaced (1.651%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:49:04.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29054/40107 objects degraded (72.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:04.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:04 smithi189 ceph-mon[90992]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 838 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 29054/40107 objects degraded (72.441%); 662/40107 objects misplaced (1.651%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:49:04.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29054/40107 objects degraded (72.441%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:04.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:04.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:04.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:05.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:05.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.220 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:05.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:05.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:05.475 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:49:05.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:05.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:05.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3560648426' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:05 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3560648426' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:05 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3560648426' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:05 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:05.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:05.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:05.924 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:06.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:06.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:06.156 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:06.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:06.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:06.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:06.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:06.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:06.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:06.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[103001]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 739 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 29054/35355 objects degraded (82.178%); 662/35355 objects misplaced (1.872%); 2.4 MiB/s, 37 objects/s recovering 2024-09-13T14:49:06.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:06.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:06.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[103001]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:49:06.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[107556]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 739 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 29054/35355 objects degraded (82.178%); 662/35355 objects misplaced (1.872%); 2.4 MiB/s, 37 objects/s recovering 2024-09-13T14:49:06.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:06.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:06.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:06 smithi017 ceph-mon[107556]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:49:06.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:06.663 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:06.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:06 smithi189 ceph-mon[90992]: pgmap v1217: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 739 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 29054/35355 objects degraded (82.178%); 662/35355 objects misplaced (1.872%); 2.4 MiB/s, 37 objects/s recovering 2024-09-13T14:49:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:06 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:06 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:06.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:06 smithi189 ceph-mon[90992]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:49:06.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:06.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:06.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:06.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:07.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:07.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.045 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:07.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:07.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:07.415 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:07] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-13T14:49:07.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:07.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:07.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:07.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:08.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:08.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:08.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:08.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:08.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:08.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:08.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:08.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:08.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:08.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:49:08.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:08.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:08 smithi017 ceph-mon[103001]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 669 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 29054/31992 objects degraded (90.816%); 662/31992 objects misplaced (2.069%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:49:08.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:08 smithi017 ceph-mon[107556]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 669 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 29054/31992 objects degraded (90.816%); 662/31992 objects misplaced (2.069%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:49:08.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:08 smithi189 ceph-mon[90992]: pgmap v1219: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 669 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 29054/31992 objects degraded (90.816%); 662/31992 objects misplaced (2.069%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T14:49:08.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:08.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:08.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:08.931 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:49:08.931 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29054/31992 objects degraded (90.816%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1745,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":370,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":10664,"data_bytes":701663926,"bytes_used":8500404224,"bytes_avail":759459880960,"bytes_total":767960285184,"degraded_objects":29054,"degraded_total":31992,"degraded_ratio":0.90816454113528378,"misplaced_objects":662,"misplaced_total":31992,"misplaced_ratio":0.020692673168292073,"recovering_objects_per_sec":36,"recovering_bytes_per_sec":2417126,"recovering_keys_per_sec":0,"num_objects_recovered":369,"num_bytes_recovered":24182784,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":638},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":27,"modified":"2024-09-13T14:41:43.161273+0000","services":{}},"progress_events":{"4e5355a3-e4fb-4c1d-b87c-71e34966890a":{"message":"Global Recovery Event (9m)\n [==========================..] (remaining: 45s)","progress":0.92929291725158691,"add_to_ceph_s":true},"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:49:08.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:08.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:09.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:09.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:09.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:09.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:09.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:09.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:09.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:09.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:09.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:09.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:09.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:09.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3777242945' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:49:09.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29054/31992 objects degraded (90.816%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:09.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:09.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:09.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3777242945' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:49:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29054/31992 objects degraded (90.816%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3777242945' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:49:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29054/31992 objects degraded (90.816%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:09.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:10.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:10.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:10.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:10.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.303 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.405 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.443 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:10.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.472 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:10.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:10.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:10.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:10 smithi189 ceph-mon[90992]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 564 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 29054/26928 objects degraded (107.895%); 381/26928 objects misplaced (1.415%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:10.785 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:10 smithi017 ceph-mon[103001]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 564 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 29054/26928 objects degraded (107.895%); 381/26928 objects misplaced (1.415%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:10.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:10 smithi017 ceph-mon[107556]: pgmap v1220: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 564 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 29054/26928 objects degraded (107.895%); 381/26928 objects misplaced (1.415%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:10.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:10.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.961 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:11.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:11.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:11.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:11.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:11.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.848 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.950 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:12.162 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:49:12.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:49:12.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.264 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:12.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.396 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:12.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:12.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:12.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:12.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:12 smithi189 ceph-mon[90992]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 443 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 29054/21153 objects degraded (137.352%); 381/21153 objects misplaced (1.801%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:12.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:12.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:12.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:12.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:12 smithi017 ceph-mon[103001]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 443 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 29054/21153 objects degraded (137.352%); 381/21153 objects misplaced (1.801%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:12.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:12.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:12 smithi017 ceph-mon[107556]: pgmap v1221: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 443 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 29054/21153 objects degraded (137.352%); 381/21153 objects misplaced (1.801%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:12.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:12.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:13.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:13.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:13.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:13.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:13.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:13.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:13.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:13.464 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:13.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:13.517 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:49:13.518 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:49:13.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:13.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:13.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:13.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:13.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:14.058 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:49:14.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:14.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.091 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:49:14.091 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:14.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:14.120 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:49:14.120 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:14.144 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:49:14.144 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:14.173 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:49:14.173 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:14.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:49:14.213 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:14.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:49:14.242 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:14.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:49:14.276 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:14.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:14.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:14.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:14.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:14.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:14.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[107556]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 370 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 29054/17625 objects degraded (164.845%); 77/17625 objects misplaced (0.437%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29054/17625 objects degraded (164.845%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2174394957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[103001]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 370 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 29054/17625 objects degraded (164.845%); 77/17625 objects misplaced (0.437%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29054/17625 objects degraded (164.845%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:14.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2174394957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:14.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:14 smithi189 ceph-mon[90992]: pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 370 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 29054/17625 objects degraded (164.845%); 77/17625 objects misplaced (0.437%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:14.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29054/17625 objects degraded (164.845%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:14.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2174394957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:14.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.903 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:14.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:15.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:15.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:15.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:15.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:15.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:15.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:15.582 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:15.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:15.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:15.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:16.054 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:16.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:16.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:16.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:16.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:16.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:16.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:16.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:16.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:16 smithi189 ceph-mon[90992]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 263 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 29054/12495 objects degraded (232.525%); 77/12495 objects misplaced (0.616%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:16.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:16.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:16 smithi017 ceph-mon[103001]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 263 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 29054/12495 objects degraded (232.525%); 77/12495 objects misplaced (0.616%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:16 smithi017 ceph-mon[107556]: pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 263 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 29054/12495 objects degraded (232.525%); 77/12495 objects misplaced (0.616%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:49:16.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:49:17.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:17.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.405 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:17.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:17.542 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:17.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:17.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:17.611 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:17] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-13T14:49:17.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:17.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:17.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:17.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:17.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:18.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:18.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:18.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:18.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:18.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:18.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.488 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 91338 objects 2024-09-13T14:49:18.489 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :486.909 2024-09-13T14:49:18.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:18.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:18 smithi017 ceph-mon[103001]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 190 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 29054/8991 objects degraded (323.145%); 77/8991 objects misplaced (0.856%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:18 smithi017 ceph-mon[107556]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 190 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 29054/8991 objects degraded (323.145%); 77/8991 objects misplaced (0.856%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:18.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:18 smithi189 ceph-mon[90992]: pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 190 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 29054/8991 objects degraded (323.145%); 77/8991 objects misplaced (0.856%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:18.840 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:18.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:18.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:18.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.043 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:49:19.045 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:49:19.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:19.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:19.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:49:19.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:19.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:19 smithi017 ceph-mon[103001]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:49:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 29054/8991 objects degraded (323.145%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:19 smithi017 ceph-mon[107556]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:49:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 29054/8991 objects degraded (323.145%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:19.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:19.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:19.862 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1225,"stamp":"2024-09-13T14:49:18.502068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":137398966,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28694,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":181599,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30466,"num_bytes_recovered":1909033117,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127784,"ondisk_log_size":127784,"up":297,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6806224,"kb_used_data":1412752,"kb_used_omap":44,"kb_used_meta":5393235,"kb_avail":743154992,"statfs":{"total":767960285184,"available":760990711808,"internally_reserved":0,"allocated":1446658048,"data_stored":1430811703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5522672887},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,5,5,6,5,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-564264960,"num_objects":-8610,"num_object_clones":0,"num_object_copies":-25830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-360,"num_objects_misplaced":-662,"num_objects_unfound":0,"num_objects_dirty":-8610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8610,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1022,"num_bytes_recovered":43384832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.211130"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":625,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478096+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:49:05.478096+0000","last_peered":"2024-09-13T14:49:05.478096+0000","last_clean":"2024-09-13T14:49:05.478096+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:49:05.478096+0000","last_undegraded":"2024-09-13T14:49:05.478096+0000","last_fullsized":"2024-09-13T14:49:05.478096+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":368,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396873+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:48:50.396873+0000","last_peered":"2024-09-13T14:48:50.396873+0000","last_clean":"2024-09-13T14:48:50.396873+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:48:50.396873+0000","last_undegraded":"2024-09-13T14:48:50.396873+0000","last_fullsized":"2024-09-13T14:48:50.396873+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":297,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395280+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:48:50.395280+0000","last_peered":"2024-09-13T14:48:50.395280+0000","last_clean":"2024-09-13T14:48:50.395280+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:48:50.395280+0000","last_undegraded":"2024-09-13T14:48:50.395280+0000","last_fullsized":"2024-09-13T14:48:50.395280+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":610,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476785+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:49:05.476785+0000","last_peered":"2024-09-13T14:49:05.476785+0000","last_clean":"2024-09-13T14:49:05.476785+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:49:05.476785+0000","last_undegraded":"2024-09-13T14:49:05.476785+0000","last_fullsized":"2024-09-13T14:49:05.476785+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":611,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477385+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:49:05.477385+0000","last_peered":"2024-09-13T14:49:05.477385+0000","last_clean":"2024-09-13T14:49:05.477385+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:49:05.477385+0000","last_undegraded":"2024-09-13T14:49:05.477385+0000","last_fullsized":"2024-09-13T14:49:05.477385+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":337,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397213+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:48:50.397213+0000","last_peered":"2024-09-13T14:48:50.397213+0000","last_clean":"2024-09-13T14:48:50.397213+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:48:50.397213+0000","last_undegraded":"2024-09-13T14:48:50.397213+0000","last_fullsized":"2024-09-13T14:48:50.397213+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":625,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.480870+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:49:05.480870+0000","last_peered":"2024-09-13T14:49:05.480870+0000","last_clean":"2024-09-13T14:49:05.480870+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:49:05.480870+0000","last_undegraded":"2024-09-13T14:49:05.480870+0000","last_fullsized":"2024-09-13T14:49:05.480870+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":359,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402003+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:48:50.402003+0000","last_peered":"2024-09-13T14:48:50.402003+0000","last_clean":"2024-09-13T14:48:50.402003+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:48:50.402003+0000","last_undegraded":"2024-09-13T14:48:50.402003+0000","last_fullsized":"2024-09-13T14:48:50.402003+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"370'11222","reported_seq":25348,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:17.053086+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:49:17.053086+0000","last_peered":"2024-09-13T14:49:17.053086+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:49:17.053086+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"307'1500","ondisk_log_start":"307'1500","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9722,"log_dups_size":0,"ondisk_log_size":9722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4456448,"num_objects":68,"num_object_clones":0,"num_object_copies":204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11222,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4661,"num_bytes_recovered":305463296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-815}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":363,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397078+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:48:50.397078+0000","last_peered":"2024-09-13T14:48:50.397078+0000","last_clean":"2024-09-13T14:48:50.397078+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:48:50.397078+0000","last_undegraded":"2024-09-13T14:48:50.397078+0000","last_fullsized":"2024-09-13T14:48:50.397078+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397550+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:48:50.397550+0000","last_peered":"2024-09-13T14:48:50.397550+0000","last_clean":"2024-09-13T14:48:50.397550+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:48:50.397550+0000","last_undegraded":"2024-09-13T14:48:50.397550+0000","last_fullsized":"2024-09-13T14:48:50.397550+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"370'11447","reported_seq":21014,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:17.598567+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:49:17.598567+0000","last_peered":"2024-09-13T14:49:17.598567+0000","last_clean":"2024-09-13T14:49:17.598567+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:49:17.598567+0000","last_undegraded":"2024-09-13T14:49:17.598567+0000","last_fullsized":"2024-09-13T14:49:17.598567+0000","mapping_epoch":326,"log_start":"309'3900","ondisk_log_start":"309'3900","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2686976,"num_objects":41,"num_object_clones":0,"num_object_copies":123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11447,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":381,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.398054+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:48:50.398054+0000","last_peered":"2024-09-13T14:48:50.398054+0000","last_clean":"2024-09-13T14:48:50.398054+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:48:50.398054+0000","last_undegraded":"2024-09-13T14:48:50.398054+0000","last_fullsized":"2024-09-13T14:48:50.398054+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395780+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:48:50.395780+0000","last_peered":"2024-09-13T14:48:50.395780+0000","last_clean":"2024-09-13T14:48:50.395780+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:48:50.395780+0000","last_undegraded":"2024-09-13T14:48:50.395780+0000","last_fullsized":"2024-09-13T14:48:50.395780+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"370'11575","reported_seq":11724,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:14.103462+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:49:14.103462+0000","last_peered":"2024-09-13T14:49:14.103462+0000","last_clean":"2024-09-13T14:49:14.103462+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:49:14.103462+0000","last_undegraded":"2024-09-13T14:49:14.103462+0000","last_fullsized":"2024-09-13T14:49:14.103462+0000","mapping_epoch":303,"log_start":"307'3000","ondisk_log_start":"307'3000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8575,"log_dups_size":0,"ondisk_log_size":8575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14614528,"num_objects":223,"num_object_clones":0,"num_object_copies":669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11575,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":335,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396911+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:48:50.396911+0000","last_peered":"2024-09-13T14:48:50.396911+0000","last_clean":"2024-09-13T14:48:50.396911+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:48:50.396911+0000","last_undegraded":"2024-09-13T14:48:50.396911+0000","last_fullsized":"2024-09-13T14:48:50.396911+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":350,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402435+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:48:50.402435+0000","last_peered":"2024-09-13T14:48:50.402435+0000","last_clean":"2024-09-13T14:48:50.402435+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:48:50.402435+0000","last_undegraded":"2024-09-13T14:48:50.402435+0000","last_fullsized":"2024-09-13T14:48:50.402435+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"370'11389","reported_seq":11538,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:15.120613+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:49:15.120613+0000","last_peered":"2024-09-13T14:49:15.120613+0000","last_clean":"2024-09-13T14:49:15.120613+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:49:15.120613+0000","last_undegraded":"2024-09-13T14:49:15.120613+0000","last_fullsized":"2024-09-13T14:49:15.120613+0000","mapping_epoch":303,"log_start":"308'3800","ondisk_log_start":"308'3800","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10289152,"num_objects":157,"num_object_clones":0,"num_object_copies":471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11389,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395709+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:48:50.395709+0000","last_peered":"2024-09-13T14:48:50.395709+0000","last_clean":"2024-09-13T14:48:50.395709+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:48:50.395709+0000","last_undegraded":"2024-09-13T14:48:50.395709+0000","last_fullsized":"2024-09-13T14:48:50.395709+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":637,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476950+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:49:05.476950+0000","last_peered":"2024-09-13T14:49:05.476950+0000","last_clean":"2024-09-13T14:49:05.476950+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:49:05.476950+0000","last_undegraded":"2024-09-13T14:49:05.476950+0000","last_fullsized":"2024-09-13T14:49:05.476950+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"370'11342","reported_seq":11488,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:15.802069+0000","last_change":"2024-09-13T14:40:46.145038+0000","last_active":"2024-09-13T14:49:15.802069+0000","last_peered":"2024-09-13T14:49:15.802069+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:49:15.802069+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"309'4300","ondisk_log_start":"309'4300","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7864320,"num_objects":120,"num_object_clones":0,"num_object_copies":360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5633,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11342,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-5513}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":481,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396262+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:48:50.396262+0000","last_peered":"2024-09-13T14:48:50.396262+0000","last_clean":"2024-09-13T14:48:50.396262+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:48:50.396262+0000","last_undegraded":"2024-09-13T14:48:50.396262+0000","last_fullsized":"2024-09-13T14:48:50.396262+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":635,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.484038+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:49:05.484038+0000","last_peered":"2024-09-13T14:49:05.484038+0000","last_clean":"2024-09-13T14:49:05.484038+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:49:05.484038+0000","last_undegraded":"2024-09-13T14:49:05.484038+0000","last_fullsized":"2024-09-13T14:49:05.484038+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.398502+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:48:50.398502+0000","last_peered":"2024-09-13T14:48:50.398502+0000","last_clean":"2024-09-13T14:48:50.398502+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:48:50.398502+0000","last_undegraded":"2024-09-13T14:48:50.398502+0000","last_fullsized":"2024-09-13T14:48:50.398502+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"370'10935","reported_seq":11084,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:15.121186+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:49:15.121186+0000","last_peered":"2024-09-13T14:49:15.121186+0000","last_clean":"2024-09-13T14:49:15.121186+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:49:15.121186+0000","last_undegraded":"2024-09-13T14:49:15.121186+0000","last_fullsized":"2024-09-13T14:49:15.121186+0000","mapping_epoch":303,"log_start":"308'3400","ondisk_log_start":"308'3400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10935,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":185,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397619+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:48:50.397619+0000","last_peered":"2024-09-13T14:48:50.397619+0000","last_clean":"2024-09-13T14:48:50.397619+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:48:50.397619+0000","last_undegraded":"2024-09-13T14:48:50.397619+0000","last_fullsized":"2024-09-13T14:48:50.397619+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":371,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.401934+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:48:50.401934+0000","last_peered":"2024-09-13T14:48:50.401934+0000","last_clean":"2024-09-13T14:48:50.401934+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:48:50.401934+0000","last_undegraded":"2024-09-13T14:48:50.401934+0000","last_fullsized":"2024-09-13T14:48:50.401934+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396355+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:48:50.396355+0000","last_peered":"2024-09-13T14:48:50.396355+0000","last_clean":"2024-09-13T14:48:50.396355+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:48:50.396355+0000","last_undegraded":"2024-09-13T14:48:50.396355+0000","last_fullsized":"2024-09-13T14:48:50.396355+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"370'11366","reported_seq":27650,"reported_epoch":370,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:17.725058+0000","last_change":"2024-09-13T14:42:17.600734+0000","last_active":"2024-09-13T14:49:17.725058+0000","last_peered":"2024-09-13T14:49:17.725058+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:41:02.155225+0000","last_became_peered":"2024-09-13T14:41:02.155225+0000","last_unstale":"2024-09-13T14:49:17.725058+0000","last_undegraded":"2024-09-13T14:40:57.039797+0000","last_fullsized":"2024-09-13T14:40:45.693249+0000","mapping_epoch":317,"log_start":"309'4000","ondisk_log_start":"309'4000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:14:42.047443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3014656,"num_objects":46,"num_object_clones":0,"num_object_copies":138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11366,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5951,"num_bytes_recovered":366411776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,4","objects":19}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395580+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:48:50.395580+0000","last_peered":"2024-09-13T14:48:50.395580+0000","last_clean":"2024-09-13T14:48:50.395580+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:48:50.395580+0000","last_undegraded":"2024-09-13T14:48:50.395580+0000","last_fullsized":"2024-09-13T14:48:50.395580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396478+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:48:50.396478+0000","last_peered":"2024-09-13T14:48:50.396478+0000","last_clean":"2024-09-13T14:48:50.396478+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:48:50.396478+0000","last_undegraded":"2024-09-13T14:48:50.396478+0000","last_fullsized":"2024-09-13T14:48:50.396478+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"370'10979","reported_seq":11128,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:15.104160+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:49:15.104160+0000","last_peered":"2024-09-13T14:49:15.104160+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:49:15.104160+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"308'3400","ondisk_log_start":"308'3400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7579,"log_dups_size":0,"ondisk_log_size":7579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11730944,"num_objects":179,"num_object_clones":0,"num_object_copies":537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10979,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-5287}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":178,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400143+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:48:50.400143+0000","last_peered":"2024-09-13T14:48:50.400143+0000","last_clean":"2024-09-13T14:48:50.400143+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:48:50.400143+0000","last_undegraded":"2024-09-13T14:48:50.400143+0000","last_fullsized":"2024-09-13T14:48:50.400143+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":627,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.484067+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:49:05.484067+0000","last_peered":"2024-09-13T14:49:05.484067+0000","last_clean":"2024-09-13T14:49:05.484067+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:49:05.484067+0000","last_undegraded":"2024-09-13T14:49:05.484067+0000","last_fullsized":"2024-09-13T14:49:05.484067+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":589,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.470151+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:49:05.470151+0000","last_peered":"2024-09-13T14:49:05.470151+0000","last_clean":"2024-09-13T14:49:05.470151+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:49:05.470151+0000","last_undegraded":"2024-09-13T14:49:05.470151+0000","last_fullsized":"2024-09-13T14:49:05.470151+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"370'11178","reported_seq":11260,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:17.608625+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:49:17.608625+0000","last_peered":"2024-09-13T14:49:17.608625+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:49:17.608625+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"308'3600","ondisk_log_start":"308'3600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11178,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-5454}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":472,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397562+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:48:50.397562+0000","last_peered":"2024-09-13T14:48:50.397562+0000","last_clean":"2024-09-13T14:48:50.397562+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:48:50.397562+0000","last_undegraded":"2024-09-13T14:48:50.397562+0000","last_fullsized":"2024-09-13T14:48:50.397562+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":894,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395073+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:48:50.395073+0000","last_peered":"2024-09-13T14:48:50.395073+0000","last_clean":"2024-09-13T14:48:50.395073+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:48:50.395073+0000","last_undegraded":"2024-09-13T14:48:50.395073+0000","last_fullsized":"2024-09-13T14:48:50.395073+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":581,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478357+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:49:05.478357+0000","last_peered":"2024-09-13T14:49:05.478357+0000","last_clean":"2024-09-13T14:49:05.478357+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:49:05.478357+0000","last_undegraded":"2024-09-13T14:49:05.478357+0000","last_fullsized":"2024-09-13T14:49:05.478357+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":336,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.404874+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:48:50.404874+0000","last_peered":"2024-09-13T14:48:50.404874+0000","last_clean":"2024-09-13T14:48:50.404874+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:48:50.404874+0000","last_undegraded":"2024-09-13T14:48:50.404874+0000","last_fullsized":"2024-09-13T14:48:50.404874+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"370'11398","reported_seq":11544,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:15.114640+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:49:15.114640+0000","last_peered":"2024-09-13T14:49:15.114640+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:49:15.114640+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"308'3800","ondisk_log_start":"308'3800","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11398,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-5530}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":434,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.474418+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:49:05.474418+0000","last_peered":"2024-09-13T14:49:05.474418+0000","last_clean":"2024-09-13T14:49:05.474418+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:49:05.474418+0000","last_undegraded":"2024-09-13T14:49:05.474418+0000","last_fullsized":"2024-09-13T14:49:05.474418+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":760,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397491+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:48:50.397491+0000","last_peered":"2024-09-13T14:48:50.397491+0000","last_clean":"2024-09-13T14:48:50.397491+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:48:50.397491+0000","last_undegraded":"2024-09-13T14:48:50.397491+0000","last_fullsized":"2024-09-13T14:48:50.397491+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":630,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477155+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:49:05.477155+0000","last_peered":"2024-09-13T14:49:05.477155+0000","last_clean":"2024-09-13T14:49:05.477155+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:49:05.477155+0000","last_undegraded":"2024-09-13T14:49:05.477155+0000","last_fullsized":"2024-09-13T14:49:05.477155+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":634,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478376+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:49:05.478376+0000","last_peered":"2024-09-13T14:49:05.478376+0000","last_clean":"2024-09-13T14:49:05.478376+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:49:05.478376+0000","last_undegraded":"2024-09-13T14:49:05.478376+0000","last_fullsized":"2024-09-13T14:49:05.478376+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"370'11533","reported_seq":21221,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:15.821214+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:49:15.821214+0000","last_peered":"2024-09-13T14:49:15.821214+0000","last_clean":"2024-09-13T14:49:15.821214+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:49:15.821214+0000","last_undegraded":"2024-09-13T14:49:15.821214+0000","last_fullsized":"2024-09-13T14:49:15.821214+0000","mapping_epoch":330,"log_start":"309'4500","ondisk_log_start":"309'4500","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8585216,"num_objects":131,"num_object_clones":0,"num_object_copies":393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11533,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":501,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396577+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:48:50.396577+0000","last_peered":"2024-09-13T14:48:50.396577+0000","last_clean":"2024-09-13T14:48:50.396577+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:48:50.396577+0000","last_undegraded":"2024-09-13T14:48:50.396577+0000","last_fullsized":"2024-09-13T14:48:50.396577+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":598,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478110+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:49:05.478110+0000","last_peered":"2024-09-13T14:49:05.478110+0000","last_clean":"2024-09-13T14:49:05.478110+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:49:05.478110+0000","last_undegraded":"2024-09-13T14:49:05.478110+0000","last_fullsized":"2024-09-13T14:49:05.478110+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":567,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469829+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:49:05.469829+0000","last_peered":"2024-09-13T14:49:05.469829+0000","last_clean":"2024-09-13T14:49:05.469829+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:49:05.469829+0000","last_undegraded":"2024-09-13T14:49:05.469829+0000","last_fullsized":"2024-09-13T14:49:05.469829+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":119,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395207+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:48:50.395207+0000","last_peered":"2024-09-13T14:48:50.395207+0000","last_clean":"2024-09-13T14:48:50.395207+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:48:50.395207+0000","last_undegraded":"2024-09-13T14:48:50.395207+0000","last_fullsized":"2024-09-13T14:48:50.395207+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"370'11108","reported_seq":11256,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:15.712420+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:49:15.712420+0000","last_peered":"2024-09-13T14:49:15.712420+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:49:15.712420+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"309'4100","ondisk_log_start":"309'4100","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995392,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11108,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-5387}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":178,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.394966+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:48:50.394966+0000","last_peered":"2024-09-13T14:48:50.394966+0000","last_clean":"2024-09-13T14:48:50.394966+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:48:50.394966+0000","last_undegraded":"2024-09-13T14:48:50.394966+0000","last_fullsized":"2024-09-13T14:48:50.394966+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":985,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.400217+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:48:50.400217+0000","last_peered":"2024-09-13T14:48:50.400217+0000","last_clean":"2024-09-13T14:48:50.400217+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:48:50.400217+0000","last_undegraded":"2024-09-13T14:48:50.400217+0000","last_fullsized":"2024-09-13T14:48:50.400217+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397053+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:48:50.397053+0000","last_peered":"2024-09-13T14:48:50.397053+0000","last_clean":"2024-09-13T14:48:50.397053+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:48:50.397053+0000","last_undegraded":"2024-09-13T14:48:50.397053+0000","last_fullsized":"2024-09-13T14:48:50.397053+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":587,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.468701+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:49:05.468701+0000","last_peered":"2024-09-13T14:49:05.468701+0000","last_clean":"2024-09-13T14:49:05.468701+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:49:05.468701+0000","last_undegraded":"2024-09-13T14:49:05.468701+0000","last_fullsized":"2024-09-13T14:49:05.468701+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"370'11142","reported_seq":20571,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:14.072626+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:49:14.072626+0000","last_peered":"2024-09-13T14:49:14.072626+0000","last_clean":"2024-09-13T14:49:14.072626+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:49:14.072626+0000","last_undegraded":"2024-09-13T14:49:14.072626+0000","last_fullsized":"2024-09-13T14:49:14.072626+0000","mapping_epoch":337,"log_start":"307'2500","ondisk_log_start":"307'2500","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8642,"log_dups_size":0,"ondisk_log_size":8642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14286848,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11142,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":369,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402782+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:48:50.402782+0000","last_peered":"2024-09-13T14:48:50.402782+0000","last_clean":"2024-09-13T14:48:50.402782+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:48:50.402782+0000","last_undegraded":"2024-09-13T14:48:50.402782+0000","last_fullsized":"2024-09-13T14:48:50.402782+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395399+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:48:50.395399+0000","last_peered":"2024-09-13T14:48:50.395399+0000","last_clean":"2024-09-13T14:48:50.395399+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:48:50.395399+0000","last_undegraded":"2024-09-13T14:48:50.395399+0000","last_fullsized":"2024-09-13T14:48:50.395399+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1166,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.468905+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:49:05.468905+0000","last_peered":"2024-09-13T14:49:05.468905+0000","last_clean":"2024-09-13T14:49:05.468905+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:49:05.468905+0000","last_undegraded":"2024-09-13T14:49:05.468905+0000","last_fullsized":"2024-09-13T14:49:05.468905+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":568,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469865+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:49:05.469865+0000","last_peered":"2024-09-13T14:49:05.469865+0000","last_clean":"2024-09-13T14:49:05.469865+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:49:05.469865+0000","last_undegraded":"2024-09-13T14:49:05.469865+0000","last_fullsized":"2024-09-13T14:49:05.469865+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"370'11062","reported_seq":11211,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:15.816046+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:49:15.816046+0000","last_peered":"2024-09-13T14:49:15.816046+0000","last_clean":"2024-09-13T14:49:15.816046+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:49:15.816046+0000","last_undegraded":"2024-09-13T14:49:15.816046+0000","last_fullsized":"2024-09-13T14:49:15.816046+0000","mapping_epoch":303,"log_start":"309'4000","ondisk_log_start":"309'4000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9306112,"num_objects":142,"num_object_clones":0,"num_object_copies":426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11062,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":369,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402993+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:48:50.402993+0000","last_peered":"2024-09-13T14:48:50.402993+0000","last_clean":"2024-09-13T14:48:50.402993+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:48:50.402993+0000","last_undegraded":"2024-09-13T14:48:50.402993+0000","last_fullsized":"2024-09-13T14:48:50.402993+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":577,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.468804+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:49:05.468804+0000","last_peered":"2024-09-13T14:49:05.468804+0000","last_clean":"2024-09-13T14:49:05.468804+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:49:05.468804+0000","last_undegraded":"2024-09-13T14:49:05.468804+0000","last_fullsized":"2024-09-13T14:49:05.468804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":764,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396985+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:48:50.396985+0000","last_peered":"2024-09-13T14:48:50.396985+0000","last_clean":"2024-09-13T14:48:50.396985+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:48:50.396985+0000","last_undegraded":"2024-09-13T14:48:50.396985+0000","last_fullsized":"2024-09-13T14:48:50.396985+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":613,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476894+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:49:05.476894+0000","last_peered":"2024-09-13T14:49:05.476894+0000","last_clean":"2024-09-13T14:49:05.476894+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:49:05.476894+0000","last_undegraded":"2024-09-13T14:49:05.476894+0000","last_fullsized":"2024-09-13T14:49:05.476894+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"370'11487","reported_seq":11636,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:14.098785+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:49:14.098785+0000","last_peered":"2024-09-13T14:49:14.098785+0000","last_clean":"2024-09-13T14:49:14.098785+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:49:14.098785+0000","last_undegraded":"2024-09-13T14:49:14.098785+0000","last_fullsized":"2024-09-13T14:49:14.098785+0000","mapping_epoch":303,"log_start":"307'2900","ondisk_log_start":"307'2900","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8587,"log_dups_size":0,"ondisk_log_size":8587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13303808,"num_objects":203,"num_object_clones":0,"num_object_copies":609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11487,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396166+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:48:50.396166+0000","last_peered":"2024-09-13T14:48:50.396166+0000","last_clean":"2024-09-13T14:48:50.396166+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:48:50.396166+0000","last_undegraded":"2024-09-13T14:48:50.396166+0000","last_fullsized":"2024-09-13T14:48:50.396166+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":613,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.481236+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:49:05.481236+0000","last_peered":"2024-09-13T14:49:05.481236+0000","last_clean":"2024-09-13T14:49:05.481236+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:49:05.481236+0000","last_undegraded":"2024-09-13T14:49:05.481236+0000","last_fullsized":"2024-09-13T14:49:05.481236+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":953,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469402+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:49:05.469402+0000","last_peered":"2024-09-13T14:49:05.469402+0000","last_clean":"2024-09-13T14:49:05.469402+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:49:05.469402+0000","last_undegraded":"2024-09-13T14:49:05.469402+0000","last_fullsized":"2024-09-13T14:49:05.469402+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":611,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.483899+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:49:05.483899+0000","last_peered":"2024-09-13T14:49:05.483899+0000","last_clean":"2024-09-13T14:49:05.483899+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:49:05.483899+0000","last_undegraded":"2024-09-13T14:49:05.483899+0000","last_fullsized":"2024-09-13T14:49:05.483899+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"370'11489","reported_seq":22870,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:17.941491+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:49:17.941491+0000","last_peered":"2024-09-13T14:49:17.941491+0000","last_clean":"2024-09-13T14:49:17.941491+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:49:17.941491+0000","last_undegraded":"2024-09-13T14:49:17.941491+0000","last_fullsized":"2024-09-13T14:49:17.941491+0000","mapping_epoch":332,"log_start":"307'2300","ondisk_log_start":"307'2300","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9189,"log_dups_size":0,"ondisk_log_size":9189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572888,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11489,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":465,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477951+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:49:05.477951+0000","last_peered":"2024-09-13T14:49:05.477951+0000","last_clean":"2024-09-13T14:49:05.477951+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:49:05.477951+0000","last_undegraded":"2024-09-13T14:49:05.477951+0000","last_fullsized":"2024-09-13T14:49:05.477951+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":648,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477275+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:49:05.477275+0000","last_peered":"2024-09-13T14:49:05.477275+0000","last_clean":"2024-09-13T14:49:05.477275+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:49:05.477275+0000","last_undegraded":"2024-09-13T14:49:05.477275+0000","last_fullsized":"2024-09-13T14:49:05.477275+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":788,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402874+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:48:50.402874+0000","last_peered":"2024-09-13T14:48:50.402874+0000","last_clean":"2024-09-13T14:48:50.402874+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:48:50.402874+0000","last_undegraded":"2024-09-13T14:48:50.402874+0000","last_fullsized":"2024-09-13T14:48:50.402874+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395680+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:48:50.395680+0000","last_peered":"2024-09-13T14:48:50.395680+0000","last_clean":"2024-09-13T14:48:50.395680+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:48:50.395680+0000","last_undegraded":"2024-09-13T14:48:50.395680+0000","last_fullsized":"2024-09-13T14:48:50.395680+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.397136+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:48:50.397136+0000","last_peered":"2024-09-13T14:48:50.397136+0000","last_clean":"2024-09-13T14:48:50.397136+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:48:50.397136+0000","last_undegraded":"2024-09-13T14:48:50.397136+0000","last_fullsized":"2024-09-13T14:48:50.397136+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":79,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396600+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:48:50.396600+0000","last_peered":"2024-09-13T14:48:50.396600+0000","last_clean":"2024-09-13T14:48:50.396600+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:48:50.396600+0000","last_undegraded":"2024-09-13T14:48:50.396600+0000","last_fullsized":"2024-09-13T14:48:50.396600+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":565,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469720+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:49:05.469720+0000","last_peered":"2024-09-13T14:49:05.469720+0000","last_clean":"2024-09-13T14:49:05.469720+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:49:05.469720+0000","last_undegraded":"2024-09-13T14:49:05.469720+0000","last_fullsized":"2024-09-13T14:49:05.469720+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":369,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396769+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:48:50.396769+0000","last_peered":"2024-09-13T14:48:50.396769+0000","last_clean":"2024-09-13T14:48:50.396769+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:48:50.396769+0000","last_undegraded":"2024-09-13T14:48:50.396769+0000","last_fullsized":"2024-09-13T14:48:50.396769+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395206+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:48:50.395206+0000","last_peered":"2024-09-13T14:48:50.395206+0000","last_clean":"2024-09-13T14:48:50.395206+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:48:50.395206+0000","last_undegraded":"2024-09-13T14:48:50.395206+0000","last_fullsized":"2024-09-13T14:48:50.395206+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":569,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477852+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:49:05.477852+0000","last_peered":"2024-09-13T14:49:05.477852+0000","last_clean":"2024-09-13T14:49:05.477852+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:49:05.477852+0000","last_undegraded":"2024-09-13T14:49:05.477852+0000","last_fullsized":"2024-09-13T14:49:05.477852+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":621,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476663+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:49:05.476663+0000","last_peered":"2024-09-13T14:49:05.476663+0000","last_clean":"2024-09-13T14:49:05.476663+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:49:05.476663+0000","last_undegraded":"2024-09-13T14:49:05.476663+0000","last_fullsized":"2024-09-13T14:49:05.476663+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":625,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.483804+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:49:05.483804+0000","last_peered":"2024-09-13T14:49:05.483804+0000","last_clean":"2024-09-13T14:49:05.483804+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:49:05.483804+0000","last_undegraded":"2024-09-13T14:49:05.483804+0000","last_fullsized":"2024-09-13T14:49:05.483804+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":367,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.402791+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:48:50.402791+0000","last_peered":"2024-09-13T14:48:50.402791+0000","last_clean":"2024-09-13T14:48:50.402791+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:48:50.402791+0000","last_undegraded":"2024-09-13T14:48:50.402791+0000","last_fullsized":"2024-09-13T14:48:50.402791+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":643,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.483977+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:49:05.483977+0000","last_peered":"2024-09-13T14:49:05.483977+0000","last_clean":"2024-09-13T14:49:05.483977+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:49:05.483977+0000","last_undegraded":"2024-09-13T14:49:05.483977+0000","last_fullsized":"2024-09-13T14:49:05.483977+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":613,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.477524+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:49:05.477524+0000","last_peered":"2024-09-13T14:49:05.477524+0000","last_clean":"2024-09-13T14:49:05.477524+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:49:05.477524+0000","last_undegraded":"2024-09-13T14:49:05.477524+0000","last_fullsized":"2024-09-13T14:49:05.477524+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":590,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469587+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:49:05.469587+0000","last_peered":"2024-09-13T14:49:05.469587+0000","last_clean":"2024-09-13T14:49:05.469587+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:49:05.469587+0000","last_undegraded":"2024-09-13T14:49:05.469587+0000","last_fullsized":"2024-09-13T14:49:05.469587+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":639,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478545+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:49:05.478545+0000","last_peered":"2024-09-13T14:49:05.478545+0000","last_clean":"2024-09-13T14:49:05.478545+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:49:05.478545+0000","last_undegraded":"2024-09-13T14:49:05.478545+0000","last_fullsized":"2024-09-13T14:49:05.478545+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":68,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403161+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:48:50.403161+0000","last_peered":"2024-09-13T14:48:50.403161+0000","last_clean":"2024-09-13T14:48:50.403161+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:48:50.403161+0000","last_undegraded":"2024-09-13T14:48:50.403161+0000","last_fullsized":"2024-09-13T14:48:50.403161+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":535,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.469028+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:49:05.469028+0000","last_peered":"2024-09-13T14:49:05.469028+0000","last_clean":"2024-09-13T14:49:05.469028+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:49:05.469028+0000","last_undegraded":"2024-09-13T14:49:05.469028+0000","last_fullsized":"2024-09-13T14:49:05.469028+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":617,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476829+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:49:05.476829+0000","last_peered":"2024-09-13T14:49:05.476829+0000","last_clean":"2024-09-13T14:49:05.476829+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:49:05.476829+0000","last_undegraded":"2024-09-13T14:49:05.476829+0000","last_fullsized":"2024-09-13T14:49:05.476829+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.396989+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:48:50.396989+0000","last_peered":"2024-09-13T14:48:50.396989+0000","last_clean":"2024-09-13T14:48:50.396989+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:48:50.396989+0000","last_undegraded":"2024-09-13T14:48:50.396989+0000","last_fullsized":"2024-09-13T14:48:50.396989+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":335,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395778+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:48:50.395778+0000","last_peered":"2024-09-13T14:48:50.395778+0000","last_clean":"2024-09-13T14:48:50.395778+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:48:50.395778+0000","last_undegraded":"2024-09-13T14:48:50.395778+0000","last_fullsized":"2024-09-13T14:48:50.395778+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":68,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.403089+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:48:50.403089+0000","last_peered":"2024-09-13T14:48:50.403089+0000","last_clean":"2024-09-13T14:48:50.403089+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:48:50.403089+0000","last_undegraded":"2024-09-13T14:48:50.403089+0000","last_fullsized":"2024-09-13T14:48:50.403089+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":301,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395361+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:48:50.395361+0000","last_peered":"2024-09-13T14:48:50.395361+0000","last_clean":"2024-09-13T14:48:50.395361+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:48:50.395361+0000","last_undegraded":"2024-09-13T14:48:50.395361+0000","last_fullsized":"2024-09-13T14:48:50.395361+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":635,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478198+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:49:05.478198+0000","last_peered":"2024-09-13T14:49:05.478198+0000","last_clean":"2024-09-13T14:49:05.478198+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:49:05.478198+0000","last_undegraded":"2024-09-13T14:49:05.478198+0000","last_fullsized":"2024-09-13T14:49:05.478198+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":622,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.478941+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:49:05.478941+0000","last_peered":"2024-09-13T14:49:05.478941+0000","last_clean":"2024-09-13T14:49:05.478941+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:49:05.478941+0000","last_undegraded":"2024-09-13T14:49:05.478941+0000","last_fullsized":"2024-09-13T14:49:05.478941+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T14:48:50.395447+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:48:50.395447+0000","last_peered":"2024-09-13T14:48:50.395447+0000","last_clean":"2024-09-13T14:48:50.395447+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:48:50.395447+0000","last_undegraded":"2024-09-13T14:48:50.395447+0000","last_fullsized":"2024-09-13T14:48:50.395447+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":627,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T14:49:05.476630+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:49:05.476630+0000","last_peered":"2024-09-13T14:49:05.476630+0000","last_clean":"2024-09-13T14:49:05.476630+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:49:05.476630+0000","last_undegraded":"2024-09-13T14:49:05.476630+0000","last_fullsized":"2024-09-13T14:49:05.476630+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":132710424,"num_objects":2026,"num_object_clones":0,"num_object_copies":6078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28694,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2026,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180652,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30409,"num_bytes_recovered":1908867072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1328623616,"data_stored":1328611400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125652,"ondisk_log_size":125652,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397958,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915612,"kb_used_data":27924,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":92829540,"statfs":{"total":95995035648,"available":95057448960,"internally_reserved":0,"allocated":28594176,"data_stored":26597827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594186,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1280072,"kb_used_data":377152,"kb_used_omap":1,"kb_used_meta":902782,"kb_avail":92465080,"statfs":{"total":95995035648,"available":94684241920,"internally_reserved":0,"allocated":386203648,"data_stored":384227053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":924449112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665640,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":350216,"kb_used_data":25984,"kb_used_omap":9,"kb_used_meta":324214,"kb_avail":93394936,"statfs":{"total":95995035648,"available":95636414464,"internally_reserved":0,"allocated":26607616,"data_stored":24627352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":331995336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,3,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986642,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921940,"kb_used_data":30412,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":92823212,"statfs":{"total":95995035648,"available":95050969088,"internally_reserved":0,"allocated":31141888,"data_stored":29153637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697201,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188420,"kb_used_data":416380,"kb_used_omap":4,"kb_used_meta":772027,"kb_avail":92556732,"statfs":{"total":95995035648,"available":94778093568,"internally_reserved":0,"allocated":426373120,"data_stored":424397009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":790555723},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860728,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":984716,"kb_used_data":349700,"kb_used_omap":8,"kb_used_meta":634999,"kb_avail":92760436,"statfs":{"total":95995035648,"available":94986686464,"internally_reserved":0,"allocated":358092800,"data_stored":356109035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":650239820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020399,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":424896,"kb_used_data":85240,"kb_used_omap":9,"kb_used_meta":339638,"kb_avail":93320256,"statfs":{"total":95995035648,"available":95559942144,"internally_reserved":0,"allocated":87285760,"data_stored":85313122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":347790149},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187789,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":740352,"kb_used_data":99960,"kb_used_omap":7,"kb_used_meta":640376,"kb_avail":93004800,"statfs":{"total":95995035648,"available":95236915200,"internally_reserved":0,"allocated":102359040,"data_stored":100386668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":655745355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":83886080,"data_stored":83886080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":74776576,"data_stored":74776576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":337907712,"data_stored":337903640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":416743424,"data_stored":416743424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12058624,"data_stored":12058624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":13373440,"data_stored":13369368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":377552896,"data_stored":377552896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":12324864,"data_stored":12320792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:49:19.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:19 smithi189 ceph-mon[90992]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:49:20.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 29054/8991 objects degraded (323.145%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:20.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:20.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:20.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:20.261 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-13T14:49:20.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-13T14:49:20.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:20.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:20.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[107556]: from='client.36780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[107556]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 91 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 28694/4230 objects degraded (678.345%); 2.4 MiB/s, 74 objects/s recovering 2024-09-13T14:49:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[107556]: osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:49:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3963776353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[103001]: from='client.36780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[103001]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 91 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 28694/4230 objects degraded (678.345%); 2.4 MiB/s, 74 objects/s recovering 2024-09-13T14:49:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[103001]: osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:49:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3963776353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:20 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:20.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:20.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-mon[90992]: from='client.36780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-mon[90992]: pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 91 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 28694/4230 objects degraded (678.345%); 2.4 MiB/s, 74 objects/s recovering 2024-09-13T14:49:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-mon[90992]: osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:49:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3963776353' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:20 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:21.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:21.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:21.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:21.137 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:21.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:21.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:21.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:21.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:21.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:21.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:21.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:21.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:21.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.881 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[103001]: osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:49:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:21 smithi017 ceph-mon[107556]: osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:49:22.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:22.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:21 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:22.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:22.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:21 smithi189 ceph-mon[90992]: osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:49:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:22.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:22.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:22.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:22.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:22.342 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:49:22.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:22.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:22.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:22.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:22 smithi017 ceph-mon[107556]: pgmap v1229: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 18 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 28459/750 objects degraded (3794.533%); 821 KiB/s, 111 objects/s recovering 2024-09-13T14:49:22.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:22 smithi017 ceph-mon[103001]: pgmap v1229: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 18 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 28459/750 objects degraded (3794.533%); 821 KiB/s, 111 objects/s recovering 2024-09-13T14:49:22.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:22.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:22.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:22.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:49:22.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:22.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:22 smithi189 ceph-mon[90992]: pgmap v1229: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 18 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 28459/750 objects degraded (3794.533%); 821 KiB/s, 111 objects/s recovering 2024-09-13T14:49:23.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:23.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:23.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:23.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:23.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.513 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:23.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:23.540 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:23.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:23.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:23.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:23.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:23.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.017 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:24.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:24.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:24.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:24.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:24.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:24.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:24.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:24 smithi017 ceph-mon[103001]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28432/84 objects degraded (33847.619%); 821 KiB/s, 116 objects/s recovering 2024-09-13T14:49:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:24 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 28432/84 objects degraded (33847.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:24 smithi017 ceph-mon[107556]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28432/84 objects degraded (33847.619%); 821 KiB/s, 116 objects/s recovering 2024-09-13T14:49:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:24 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 28432/84 objects degraded (33847.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:24.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:24 smithi189 ceph-mon[90992]: pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 28432/84 objects degraded (33847.619%); 821 KiB/s, 116 objects/s recovering 2024-09-13T14:49:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:24 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 28432/84 objects degraded (33847.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:25.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.263 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:25.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:25.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:25.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:25.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:49:25.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:25.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:25.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.668 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:25.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:25.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:25 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:25 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:26.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:25 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:26.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:26.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:26.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:26.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:26.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:26.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:26.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:26.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:26.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:26.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:26.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.871 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:26 smithi189 ceph-mon[90992]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 28432/84 objects degraded (33847.619%); 0 B/s, 38 objects/s recovering 2024-09-13T14:49:26.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:26.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:26.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:26 smithi017 ceph-mon[103001]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 28432/84 objects degraded (33847.619%); 0 B/s, 38 objects/s recovering 2024-09-13T14:49:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:26 smithi017 ceph-mon[107556]: pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 298 op/s; 28432/84 objects degraded (33847.619%); 0 B/s, 38 objects/s recovering 2024-09-13T14:49:27.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:27.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:27.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:27.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:27.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:27.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:27.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:27.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:27.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:27.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:27.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:27.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:27] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-09-13T14:49:27.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:27.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:27.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:27.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:28.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:28.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:28.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:28.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:28.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:28.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:28.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:28.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[103001]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 99 objects/s recovering 2024-09-13T14:49:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:28 smithi017 ceph-mon[107556]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 99 objects/s recovering 2024-09-13T14:49:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:28 smithi189 ceph-mon[90992]: pgmap v1232: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 99 objects/s recovering 2024-09-13T14:49:28.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:28.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:28.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:28.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:28.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:28.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:29.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:29.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:29.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:29.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.278 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:29.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:29.379 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:29.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 27901/84 objects degraded (33215.476%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 27901/84 objects degraded (33215.476%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:29.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 27901/84 objects degraded (33215.476%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:29.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:29.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:29.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:29.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:29.874 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:29.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:29.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:30.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:30.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:30.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:30.247 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:30.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:30.259 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:30 smithi189 ceph-mon[90992]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 80 objects/s recovering 2024-09-13T14:49:30.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:30.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:30.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:30 smithi017 ceph-mon[103001]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 80 objects/s recovering 2024-09-13T14:49:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:30 smithi017 ceph-mon[107556]: pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 27901/84 objects degraded (33215.476%); 0 B/s, 80 objects/s recovering 2024-09-13T14:49:30.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.516 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:30.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:30.639 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:30.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:30.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:30.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:30.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:30.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:31.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:31.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.380 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:31.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:31.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:31.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:31.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:31.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:31.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:31.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.107 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:32.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:32.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:32.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:32.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:32.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.627 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:32.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[103001]: pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 128 objects/s recovering 2024-09-13T14:49:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[107556]: pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 128 objects/s recovering 2024-09-13T14:49:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:32.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:32 smithi189 ceph-mon[90992]: pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 128 objects/s recovering 2024-09-13T14:49:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:32.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:33.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:33.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:33.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:33.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:33.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:33.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:33.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:33.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:33.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:33.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:33.968 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:33.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:34.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:34.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:34.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:34.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:34.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:34.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:34.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:34.485 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:34.587 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:34.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:34.709 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.731 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:34 smithi189 ceph-mon[90992]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 95 objects/s recovering 2024-09-13T14:49:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 27309/84 objects degraded (32510.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:34.833 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:34.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:34 smithi017 ceph-mon[103001]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 95 objects/s recovering 2024-09-13T14:49:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 27309/84 objects degraded (32510.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:34 smithi017 ceph-mon[107556]: pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 18 op/s; 27309/84 objects degraded (32510.714%); 0 B/s, 95 objects/s recovering 2024-09-13T14:49:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 27309/84 objects degraded (32510.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:34.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:35.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:35.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:35.365 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:35.453 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:35.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:35.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:35.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:35.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:35.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:35.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:35.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:36.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:36.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:36.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:36.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:36.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:36.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:36.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:36.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:36.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:36 smithi017 ceph-mon[103001]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 27309/84 objects degraded (32510.714%); 0 B/s, 93 objects/s recovering 2024-09-13T14:49:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:36 smithi017 ceph-mon[107556]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 27309/84 objects degraded (32510.714%); 0 B/s, 93 objects/s recovering 2024-09-13T14:49:36.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:36 smithi189 ceph-mon[90992]: pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 27309/84 objects degraded (32510.714%); 0 B/s, 93 objects/s recovering 2024-09-13T14:49:36.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:36.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:37.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:37.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:37.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:37.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:37] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-09-13T14:49:37.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:37.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:37.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:37.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:37.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.950 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:49:37.950 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:49:37.950 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:49:37.950 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:49:37.950 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:49:37.955 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:38.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.203 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:38.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:38.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:38 smithi189 ceph-mon[90992]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 151 objects/s recovering 2024-09-13T14:49:38.799 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:38.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:38 smithi017 ceph-mon[103001]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 151 objects/s recovering 2024-09-13T14:49:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:38 smithi017 ceph-mon[107556]: pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 151 objects/s recovering 2024-09-13T14:49:39.045 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:39.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:39.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.294 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:39.315 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:39.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.541 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:39.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:39.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:39.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:39.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 26619/84 objects degraded (31689.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 26619/84 objects degraded (31689.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:40.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 26619/84 objects degraded (31689.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:40.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:40.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:40.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:40.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:40.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:40.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:40.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:40.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:40.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:40.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:40.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:40 smithi017 ceph-mon[103001]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 106 objects/s recovering 2024-09-13T14:49:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:40 smithi017 ceph-mon[107556]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 106 objects/s recovering 2024-09-13T14:49:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:40 smithi189 ceph-mon[90992]: pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26619/84 objects degraded (31689.286%); 0 B/s, 106 objects/s recovering 2024-09-13T14:49:41.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:41.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:41.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:41.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:41.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:41.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.526 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:41.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:41.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:41.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:41.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:41.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:42.021 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:42.029 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:49:42.030 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:49:42.042 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:42.144 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:42.166 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:42.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:42.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:42.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:42.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:42.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:42.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:42.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:42.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:49:42.749 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:42.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:49:42.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:49:42.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:49:42.750 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:42.751 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:42.751 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:42.751 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:42.751 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:49:42.751 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:42.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:42.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:42.890 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[103001]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 177 objects/s recovering 2024-09-13T14:49:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1784719690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[107556]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 177 objects/s recovering 2024-09-13T14:49:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1784719690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:42 smithi189 ceph-mon[90992]: pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 177 objects/s recovering 2024-09-13T14:49:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1784719690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:43.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:43.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:43.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:43.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:43.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:43.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:43.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:43.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:43.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:43.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:43.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:43.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:43.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:43 smithi017 ceph-mon[103001]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:43 smithi017 ceph-mon[107556]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:43 smithi189 ceph-mon[90992]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:49:44.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:44.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:44.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:44.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.773 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:44.897 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:44.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:44 smithi017 ceph-mon[103001]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 25776/84 objects degraded (30685.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:44.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:44 smithi017 ceph-mon[107556]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:44.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 25776/84 objects degraded (30685.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:44.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:44 smithi189 ceph-mon[90992]: pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 25776/84 objects degraded (30685.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:45.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:45.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:45.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:45.249 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:45.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:45.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:45.497 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:45.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:45.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:45.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:45.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:45.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:45.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:45.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:45.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:46.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:49:46.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:46.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:46.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.360 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:46.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:46.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:46.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:46 smithi017 ceph-mon[103001]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:46.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:46 smithi017 ceph-mon[107556]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:46.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:46.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:46.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:47.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:46 smithi189 ceph-mon[90992]: pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 25776/84 objects degraded (30685.714%); 0 B/s, 127 objects/s recovering 2024-09-13T14:49:47.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:47.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:47.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:47.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:47.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.373 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:47.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:47.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:47.524 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' removed 2024-09-13T14:49:47.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:47.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:47 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:49:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:47] "GET /metrics HTTP/1.1" 200 36678 "" "Prometheus/2.51.0" 2024-09-13T14:49:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:47 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:49:47.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:47.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:47.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:47 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:49:48.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.108 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:48.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:48.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:48.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:48.254 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:48.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:48.256 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1239,"stamp":"2024-09-13T14:49:41.304236+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4688542,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25776,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6048,"num_read_kb":13186,"num_write":183625,"num_write_kb":5858700,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33384,"num_bytes_recovered":1909033117,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127710,"ondisk_log_size":127710,"up":297,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6493176,"kb_used_data":1070660,"kb_used_omap":44,"kb_used_meta":5422355,"kb_avail":743468040,"statfs":{"total":767960285184,"available":761311272960,"internally_reserved":0,"allocated":1096355840,"data_stored":1080332983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5552491767},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2125,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2125,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003184"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":632,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545449+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:49:20.545449+0000","last_peered":"2024-09-13T14:49:20.545449+0000","last_clean":"2024-09-13T14:49:20.545449+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:49:20.545449+0000","last_undegraded":"2024-09-13T14:49:20.545449+0000","last_fullsized":"2024-09-13T14:49:20.545449+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":371,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189119+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:49:20.189119+0000","last_peered":"2024-09-13T14:49:20.189119+0000","last_clean":"2024-09-13T14:49:20.189119+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:49:20.189119+0000","last_undegraded":"2024-09-13T14:49:20.189119+0000","last_fullsized":"2024-09-13T14:49:20.189119+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":300,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511050+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:49:19.511050+0000","last_peered":"2024-09-13T14:49:19.511050+0000","last_clean":"2024-09-13T14:49:19.511050+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:49:19.511050+0000","last_undegraded":"2024-09-13T14:49:19.511050+0000","last_fullsized":"2024-09-13T14:49:19.511050+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":617,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.173939+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:49:21.173939+0000","last_peered":"2024-09-13T14:49:21.173939+0000","last_clean":"2024-09-13T14:49:21.173939+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:49:21.173939+0000","last_undegraded":"2024-09-13T14:49:21.173939+0000","last_fullsized":"2024-09-13T14:49:21.173939+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":617,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174341+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:49:21.174341+0000","last_peered":"2024-09-13T14:49:21.174341+0000","last_clean":"2024-09-13T14:49:21.174341+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:49:21.174341+0000","last_undegraded":"2024-09-13T14:49:21.174341+0000","last_fullsized":"2024-09-13T14:49:21.174341+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":340,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.188937+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:49:20.188937+0000","last_peered":"2024-09-13T14:49:20.188937+0000","last_clean":"2024-09-13T14:49:20.188937+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:49:20.188937+0000","last_undegraded":"2024-09-13T14:49:20.188937+0000","last_fullsized":"2024-09-13T14:49:20.188937+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":631,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545813+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:49:20.545813+0000","last_peered":"2024-09-13T14:49:20.545813+0000","last_clean":"2024-09-13T14:49:20.545813+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:49:20.545813+0000","last_undegraded":"2024-09-13T14:49:20.545813+0000","last_fullsized":"2024-09-13T14:49:20.545813+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":362,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508871+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:49:19.508871+0000","last_peered":"2024-09-13T14:49:19.508871+0000","last_clean":"2024-09-13T14:49:19.508871+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:49:19.508871+0000","last_undegraded":"2024-09-13T14:49:19.508871+0000","last_fullsized":"2024-09-13T14:49:19.508871+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"370'11290","reported_seq":25422,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:21.174235+0000","last_change":"2024-09-13T14:40:46.060850+0000","last_active":"2024-09-13T14:49:21.174235+0000","last_peered":"2024-09-13T14:49:21.174235+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.046549+0000","last_became_peered":"2024-09-13T14:40:46.046549+0000","last_unstale":"2024-09-13T14:49:21.174235+0000","last_undegraded":"2024-09-13T14:40:45.832556+0000","last_fullsized":"2024-09-13T14:40:45.700225+0000","mapping_epoch":317,"log_start":"307'1600","ondisk_log_start":"307'1600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9690,"log_dups_size":0,"ondisk_log_size":9690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11290,"num_write_kb":361280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4661,"num_bytes_recovered":305463296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-883}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":366,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189359+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:49:20.189359+0000","last_peered":"2024-09-13T14:49:20.189359+0000","last_clean":"2024-09-13T14:49:20.189359+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:49:20.189359+0000","last_undegraded":"2024-09-13T14:49:20.189359+0000","last_fullsized":"2024-09-13T14:49:20.189359+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":285,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509011+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:49:19.509011+0000","last_peered":"2024-09-13T14:49:19.509011+0000","last_clean":"2024-09-13T14:49:19.509011+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:49:19.509011+0000","last_undegraded":"2024-09-13T14:49:19.509011+0000","last_fullsized":"2024-09-13T14:49:19.509011+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"370'11488","reported_seq":21058,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509290+0000","last_change":"2024-09-13T14:41:28.273574+0000","last_active":"2024-09-13T14:49:19.509290+0000","last_peered":"2024-09-13T14:49:19.509290+0000","last_clean":"2024-09-13T14:49:19.509290+0000","last_became_active":"2024-09-13T14:41:28.264178+0000","last_became_peered":"2024-09-13T14:41:28.264178+0000","last_unstale":"2024-09-13T14:49:19.509290+0000","last_undegraded":"2024-09-13T14:49:19.509290+0000","last_fullsized":"2024-09-13T14:49:19.509290+0000","mapping_epoch":326,"log_start":"309'3900","ondisk_log_start":"309'3900","created":303,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7588,"log_dups_size":0,"ondisk_log_size":7588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:00.341503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11488,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4735,"num_bytes_recovered":310312960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":384,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189371+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:49:20.189371+0000","last_peered":"2024-09-13T14:49:20.189371+0000","last_clean":"2024-09-13T14:49:20.189371+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:49:20.189371+0000","last_undegraded":"2024-09-13T14:49:20.189371+0000","last_fullsized":"2024-09-13T14:49:20.189371+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510047+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:49:19.510047+0000","last_peered":"2024-09-13T14:49:19.510047+0000","last_clean":"2024-09-13T14:49:19.510047+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:49:19.510047+0000","last_undegraded":"2024-09-13T14:49:19.510047+0000","last_fullsized":"2024-09-13T14:49:19.510047+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"370'11798","reported_seq":11953,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174529+0000","last_change":"2024-09-13T14:39:32.319140+0000","last_active":"2024-09-13T14:49:21.174529+0000","last_peered":"2024-09-13T14:49:21.174529+0000","last_clean":"2024-09-13T14:49:21.174529+0000","last_became_active":"2024-09-13T14:39:32.315564+0000","last_became_peered":"2024-09-13T14:39:32.315564+0000","last_unstale":"2024-09-13T14:49:21.174529+0000","last_undegraded":"2024-09-13T14:49:21.174529+0000","last_fullsized":"2024-09-13T14:49:21.174529+0000","mapping_epoch":303,"log_start":"307'3200","ondisk_log_start":"307'3200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8598,"log_dups_size":0,"ondisk_log_size":8598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:45.000641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11798,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":338,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189439+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:49:20.189439+0000","last_peered":"2024-09-13T14:49:20.189439+0000","last_clean":"2024-09-13T14:49:20.189439+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:49:20.189439+0000","last_undegraded":"2024-09-13T14:49:20.189439+0000","last_fullsized":"2024-09-13T14:49:20.189439+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":353,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509125+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:49:19.509125+0000","last_peered":"2024-09-13T14:49:19.509125+0000","last_clean":"2024-09-13T14:49:19.509125+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:49:19.509125+0000","last_undegraded":"2024-09-13T14:49:19.509125+0000","last_fullsized":"2024-09-13T14:49:19.509125+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"370'11546","reported_seq":11701,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175504+0000","last_change":"2024-09-13T14:39:32.320319+0000","last_active":"2024-09-13T14:49:21.175504+0000","last_peered":"2024-09-13T14:49:21.175504+0000","last_clean":"2024-09-13T14:49:21.175504+0000","last_became_active":"2024-09-13T14:39:32.320011+0000","last_became_peered":"2024-09-13T14:39:32.320011+0000","last_unstale":"2024-09-13T14:49:21.175504+0000","last_undegraded":"2024-09-13T14:49:21.175504+0000","last_fullsized":"2024-09-13T14:49:21.175504+0000","mapping_epoch":303,"log_start":"309'4000","ondisk_log_start":"309'4000","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:44:58.587490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11546,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":301,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510571+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:49:19.510571+0000","last_peered":"2024-09-13T14:49:19.510571+0000","last_clean":"2024-09-13T14:49:19.510571+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:49:19.510571+0000","last_undegraded":"2024-09-13T14:49:19.510571+0000","last_fullsized":"2024-09-13T14:49:19.510571+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":643,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.177407+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:49:21.177407+0000","last_peered":"2024-09-13T14:49:21.177407+0000","last_clean":"2024-09-13T14:49:21.177407+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:49:21.177407+0000","last_undegraded":"2024-09-13T14:49:21.177407+0000","last_fullsized":"2024-09-13T14:49:21.177407+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"370'11462","reported_seq":17398,"reported_epoch":373,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:40.857364+0000","last_change":"2024-09-13T14:49:17.939255+0000","last_active":"2024-09-13T14:49:40.857364+0000","last_peered":"2024-09-13T14:49:40.857364+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.133528+0000","last_became_peered":"2024-09-13T14:40:46.133528+0000","last_unstale":"2024-09-13T14:49:40.857364+0000","last_undegraded":"2024-09-13T14:40:45.741539+0000","last_fullsized":"2024-09-13T14:40:45.691119+0000","mapping_epoch":317,"log_start":"309'4400","ondisk_log_start":"309'4400","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11462,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2891,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":-2742}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":484,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.188850+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:49:20.188850+0000","last_peered":"2024-09-13T14:49:20.188850+0000","last_clean":"2024-09-13T14:49:20.188850+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:49:20.188850+0000","last_undegraded":"2024-09-13T14:49:20.188850+0000","last_fullsized":"2024-09-13T14:49:20.188850+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":641,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545240+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:49:20.545240+0000","last_peered":"2024-09-13T14:49:20.545240+0000","last_clean":"2024-09-13T14:49:20.545240+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:49:20.545240+0000","last_undegraded":"2024-09-13T14:49:20.545240+0000","last_fullsized":"2024-09-13T14:49:20.545240+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510294+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:49:19.510294+0000","last_peered":"2024-09-13T14:49:19.510294+0000","last_clean":"2024-09-13T14:49:19.510294+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:49:19.510294+0000","last_undegraded":"2024-09-13T14:49:19.510294+0000","last_fullsized":"2024-09-13T14:49:19.510294+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"370'11094","reported_seq":11249,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175188+0000","last_change":"2024-09-13T14:39:32.490974+0000","last_active":"2024-09-13T14:49:21.175188+0000","last_peered":"2024-09-13T14:49:21.175188+0000","last_clean":"2024-09-13T14:49:21.175188+0000","last_became_active":"2024-09-13T14:39:32.490380+0000","last_became_peered":"2024-09-13T14:39:32.490380+0000","last_unstale":"2024-09-13T14:49:21.175188+0000","last_undegraded":"2024-09-13T14:49:21.175188+0000","last_fullsized":"2024-09-13T14:49:21.175188+0000","mapping_epoch":303,"log_start":"308'3500","ondisk_log_start":"308'3500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:45.203465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11094,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":188,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508804+0000","last_change":"2024-09-13T14:40:44.708270+0000","last_active":"2024-09-13T14:49:19.508804+0000","last_peered":"2024-09-13T14:49:19.508804+0000","last_clean":"2024-09-13T14:49:19.508804+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:49:19.508804+0000","last_undegraded":"2024-09-13T14:49:19.508804+0000","last_fullsized":"2024-09-13T14:49:19.508804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:42.699875+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:42.699875+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":374,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508994+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:49:19.508994+0000","last_peered":"2024-09-13T14:49:19.508994+0000","last_clean":"2024-09-13T14:49:19.508994+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:49:19.508994+0000","last_undegraded":"2024-09-13T14:49:19.508994+0000","last_fullsized":"2024-09-13T14:49:19.508994+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511273+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:49:19.511273+0000","last_peered":"2024-09-13T14:49:19.511273+0000","last_clean":"2024-09-13T14:49:19.511273+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:49:19.511273+0000","last_undegraded":"2024-09-13T14:49:19.511273+0000","last_fullsized":"2024-09-13T14:49:19.511273+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"370'11412","reported_seq":27764,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.538705+0000","last_change":"2024-09-13T14:49:19.538444+0000","last_active":"2024-09-13T14:49:19.538705+0000","last_peered":"2024-09-13T14:49:19.538705+0000","last_clean":"2024-09-13T14:49:19.538705+0000","last_became_active":"2024-09-13T14:49:19.513904+0000","last_became_peered":"2024-09-13T14:49:19.513904+0000","last_unstale":"2024-09-13T14:49:19.538705+0000","last_undegraded":"2024-09-13T14:49:19.538705+0000","last_fullsized":"2024-09-13T14:49:19.538705+0000","mapping_epoch":371,"log_start":"309'4000","ondisk_log_start":"309'4000","created":303,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7412,"log_dups_size":0,"ondisk_log_size":7412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:35:56.197850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11412,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5978,"num_bytes_recovered":366411776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509905+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:49:19.509905+0000","last_peered":"2024-09-13T14:49:19.509905+0000","last_clean":"2024-09-13T14:49:19.509905+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:49:19.509905+0000","last_undegraded":"2024-09-13T14:49:19.509905+0000","last_fullsized":"2024-09-13T14:49:19.509905+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511254+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:49:19.511254+0000","last_peered":"2024-09-13T14:49:19.511254+0000","last_clean":"2024-09-13T14:49:19.511254+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:49:19.511254+0000","last_undegraded":"2024-09-13T14:49:19.511254+0000","last_fullsized":"2024-09-13T14:49:19.511254+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"370'11158","reported_seq":11313,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:21.174753+0000","last_change":"2024-09-13T14:40:46.050596+0000","last_active":"2024-09-13T14:49:21.174753+0000","last_peered":"2024-09-13T14:49:21.174753+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.037428+0000","last_became_peered":"2024-09-13T14:40:46.037428+0000","last_unstale":"2024-09-13T14:49:21.174753+0000","last_undegraded":"2024-09-13T14:40:45.736257+0000","last_fullsized":"2024-09-13T14:40:45.695438+0000","mapping_epoch":317,"log_start":"308'3600","ondisk_log_start":"308'3600","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11158,"num_write_kb":357056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":-5466}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":181,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508939+0000","last_change":"2024-09-13T14:40:44.707038+0000","last_active":"2024-09-13T14:49:19.508939+0000","last_peered":"2024-09-13T14:49:19.508939+0000","last_clean":"2024-09-13T14:49:19.508939+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:49:19.508939+0000","last_undegraded":"2024-09-13T14:49:19.508939+0000","last_fullsized":"2024-09-13T14:49:19.508939+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:39.679626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:39.679626+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":633,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.547978+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:49:20.547978+0000","last_peered":"2024-09-13T14:49:20.547978+0000","last_clean":"2024-09-13T14:49:20.547978+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:49:20.547978+0000","last_undegraded":"2024-09-13T14:49:20.547978+0000","last_fullsized":"2024-09-13T14:49:20.547978+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":595,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175660+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:49:21.175660+0000","last_peered":"2024-09-13T14:49:21.175660+0000","last_clean":"2024-09-13T14:49:21.175660+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:49:21.175660+0000","last_undegraded":"2024-09-13T14:49:21.175660+0000","last_fullsized":"2024-09-13T14:49:21.175660+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"370'11220","reported_seq":11305,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:19.509577+0000","last_change":"2024-09-13T14:40:45.973847+0000","last_active":"2024-09-13T14:49:19.509577+0000","last_peered":"2024-09-13T14:49:19.509577+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:45.954207+0000","last_became_peered":"2024-09-13T14:40:45.954207+0000","last_unstale":"2024-09-13T14:49:19.509577+0000","last_undegraded":"2024-09-13T14:40:45.725857+0000","last_fullsized":"2024-09-13T14:40:45.692204+0000","mapping_epoch":317,"log_start":"308'3700","ondisk_log_start":"308'3700","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7520,"log_dups_size":0,"ondisk_log_size":7520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:21.202739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11220,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-5496}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":475,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.188927+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:49:20.188927+0000","last_peered":"2024-09-13T14:49:20.188927+0000","last_clean":"2024-09-13T14:49:20.188927+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:49:20.188927+0000","last_undegraded":"2024-09-13T14:49:20.188927+0000","last_fullsized":"2024-09-13T14:49:20.188927+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":897,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510541+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:49:19.510541+0000","last_peered":"2024-09-13T14:49:19.510541+0000","last_clean":"2024-09-13T14:49:19.510541+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:49:19.510541+0000","last_undegraded":"2024-09-13T14:49:19.510541+0000","last_fullsized":"2024-09-13T14:49:19.510541+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":587,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.544934+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:49:20.544934+0000","last_peered":"2024-09-13T14:49:20.544934+0000","last_clean":"2024-09-13T14:49:20.544934+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:49:20.544934+0000","last_undegraded":"2024-09-13T14:49:20.544934+0000","last_fullsized":"2024-09-13T14:49:20.544934+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":339,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509568+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:49:19.509568+0000","last_peered":"2024-09-13T14:49:19.509568+0000","last_clean":"2024-09-13T14:49:19.509568+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:49:19.509568+0000","last_undegraded":"2024-09-13T14:49:19.509568+0000","last_fullsized":"2024-09-13T14:49:19.509568+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"370'11548","reported_seq":11701,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:21.174226+0000","last_change":"2024-09-13T14:40:46.049819+0000","last_active":"2024-09-13T14:49:21.174226+0000","last_peered":"2024-09-13T14:49:21.174226+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.038759+0000","last_became_peered":"2024-09-13T14:40:46.038759+0000","last_unstale":"2024-09-13T14:49:21.174226+0000","last_undegraded":"2024-09-13T14:40:45.735998+0000","last_fullsized":"2024-09-13T14:40:45.696279+0000","mapping_epoch":317,"log_start":"309'4000","ondisk_log_start":"309'4000","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11548,"num_write_kb":369536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":-5680}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":441,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175617+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:49:21.175617+0000","last_peered":"2024-09-13T14:49:21.175617+0000","last_clean":"2024-09-13T14:49:21.175617+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:49:21.175617+0000","last_undegraded":"2024-09-13T14:49:21.175617+0000","last_fullsized":"2024-09-13T14:49:21.175617+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":763,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.188918+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:49:20.188918+0000","last_peered":"2024-09-13T14:49:20.188918+0000","last_clean":"2024-09-13T14:49:20.188918+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:49:20.188918+0000","last_undegraded":"2024-09-13T14:49:20.188918+0000","last_fullsized":"2024-09-13T14:49:20.188918+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":636,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174473+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:49:21.174473+0000","last_peered":"2024-09-13T14:49:21.174473+0000","last_clean":"2024-09-13T14:49:21.174473+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:49:21.174473+0000","last_undegraded":"2024-09-13T14:49:21.174473+0000","last_fullsized":"2024-09-13T14:49:21.174473+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":640,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174794+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:49:21.174794+0000","last_peered":"2024-09-13T14:49:21.174794+0000","last_clean":"2024-09-13T14:49:21.174794+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:49:21.174794+0000","last_undegraded":"2024-09-13T14:49:21.174794+0000","last_fullsized":"2024-09-13T14:49:21.174794+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"370'11664","reported_seq":21360,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545072+0000","last_change":"2024-09-13T14:42:13.669112+0000","last_active":"2024-09-13T14:49:20.545072+0000","last_peered":"2024-09-13T14:49:20.545072+0000","last_clean":"2024-09-13T14:49:20.545072+0000","last_became_active":"2024-09-13T14:42:13.656144+0000","last_became_peered":"2024-09-13T14:42:13.656144+0000","last_unstale":"2024-09-13T14:49:20.545072+0000","last_undegraded":"2024-09-13T14:49:20.545072+0000","last_fullsized":"2024-09-13T14:49:20.545072+0000","mapping_epoch":330,"log_start":"309'4600","ondisk_log_start":"309'4600","created":303,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:22.693471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11664,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4765,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":504,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189011+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:49:20.189011+0000","last_peered":"2024-09-13T14:49:20.189011+0000","last_clean":"2024-09-13T14:49:20.189011+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:49:20.189011+0000","last_undegraded":"2024-09-13T14:49:20.189011+0000","last_fullsized":"2024-09-13T14:49:20.189011+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":604,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545019+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:49:20.545019+0000","last_peered":"2024-09-13T14:49:20.545019+0000","last_clean":"2024-09-13T14:49:20.545019+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:49:20.545019+0000","last_undegraded":"2024-09-13T14:49:20.545019+0000","last_fullsized":"2024-09-13T14:49:20.545019+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":573,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174101+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:49:21.174101+0000","last_peered":"2024-09-13T14:49:21.174101+0000","last_clean":"2024-09-13T14:49:21.174101+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:49:21.174101+0000","last_undegraded":"2024-09-13T14:49:21.174101+0000","last_fullsized":"2024-09-13T14:49:21.174101+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":122,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510228+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:49:19.510228+0000","last_peered":"2024-09-13T14:49:19.510228+0000","last_clean":"2024-09-13T14:49:19.510228+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:49:19.510228+0000","last_undegraded":"2024-09-13T14:49:19.510228+0000","last_fullsized":"2024-09-13T14:49:19.510228+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"370'11230","reported_seq":11384,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:49:20.544781+0000","last_change":"2024-09-13T14:40:46.145051+0000","last_active":"2024-09-13T14:49:20.544781+0000","last_peered":"2024-09-13T14:49:20.544781+0000","last_clean":"2024-09-13T14:39:31.301239+0000","last_became_active":"2024-09-13T14:40:46.135114+0000","last_became_peered":"2024-09-13T14:40:46.135114+0000","last_unstale":"2024-09-13T14:49:20.544781+0000","last_undegraded":"2024-09-13T14:40:45.742127+0000","last_fullsized":"2024-09-13T14:40:45.691084+0000","mapping_epoch":317,"log_start":"309'4200","ondisk_log_start":"309'4200","created":303,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11230,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-5509}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":181,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508675+0000","last_change":"2024-09-13T14:40:44.708487+0000","last_active":"2024-09-13T14:49:19.508675+0000","last_peered":"2024-09-13T14:49:19.508675+0000","last_clean":"2024-09-13T14:49:19.508675+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:49:19.508675+0000","last_undegraded":"2024-09-13T14:49:19.508675+0000","last_fullsized":"2024-09-13T14:49:19.508675+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.695836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.695836+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":988,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.508967+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:49:19.508967+0000","last_peered":"2024-09-13T14:49:19.508967+0000","last_clean":"2024-09-13T14:49:19.508967+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:49:19.508967+0000","last_undegraded":"2024-09-13T14:49:19.508967+0000","last_fullsized":"2024-09-13T14:49:19.508967+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510496+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:49:19.510496+0000","last_peered":"2024-09-13T14:49:19.510496+0000","last_clean":"2024-09-13T14:49:19.510496+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:49:19.510496+0000","last_undegraded":"2024-09-13T14:49:19.510496+0000","last_fullsized":"2024-09-13T14:49:19.510496+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":594,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174244+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:49:21.174244+0000","last_peered":"2024-09-13T14:49:21.174244+0000","last_clean":"2024-09-13T14:49:21.174244+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:49:21.174244+0000","last_undegraded":"2024-09-13T14:49:21.174244+0000","last_fullsized":"2024-09-13T14:49:21.174244+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"370'11360","reported_seq":20795,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174448+0000","last_change":"2024-09-13T14:42:49.659535+0000","last_active":"2024-09-13T14:49:21.174448+0000","last_peered":"2024-09-13T14:49:21.174448+0000","last_clean":"2024-09-13T14:49:21.174448+0000","last_became_active":"2024-09-13T14:42:49.645997+0000","last_became_peered":"2024-09-13T14:42:49.645997+0000","last_unstale":"2024-09-13T14:49:21.174448+0000","last_undegraded":"2024-09-13T14:49:21.174448+0000","last_fullsized":"2024-09-13T14:49:21.174448+0000","mapping_epoch":337,"log_start":"307'2700","ondisk_log_start":"307'2700","created":303,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8660,"log_dups_size":0,"ondisk_log_size":8660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:55.089394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11360,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4635,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":372,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509375+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:49:19.509375+0000","last_peered":"2024-09-13T14:49:19.509375+0000","last_clean":"2024-09-13T14:49:19.509375+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:49:19.509375+0000","last_undegraded":"2024-09-13T14:49:19.509375+0000","last_fullsized":"2024-09-13T14:49:19.509375+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":318,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510961+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:49:19.510961+0000","last_peered":"2024-09-13T14:49:19.510961+0000","last_clean":"2024-09-13T14:49:19.510961+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:49:19.510961+0000","last_undegraded":"2024-09-13T14:49:19.510961+0000","last_fullsized":"2024-09-13T14:49:19.510961+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1173,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174432+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:49:21.174432+0000","last_peered":"2024-09-13T14:49:21.174432+0000","last_clean":"2024-09-13T14:49:21.174432+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:49:21.174432+0000","last_undegraded":"2024-09-13T14:49:21.174432+0000","last_fullsized":"2024-09-13T14:49:21.174432+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":575,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174777+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:49:21.174777+0000","last_peered":"2024-09-13T14:49:21.174777+0000","last_clean":"2024-09-13T14:49:21.174777+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:49:21.174777+0000","last_undegraded":"2024-09-13T14:49:21.174777+0000","last_fullsized":"2024-09-13T14:49:21.174777+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"370'11204","reported_seq":11359,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545834+0000","last_change":"2024-09-13T14:39:32.489787+0000","last_active":"2024-09-13T14:49:20.545834+0000","last_peered":"2024-09-13T14:49:20.545834+0000","last_clean":"2024-09-13T14:49:20.545834+0000","last_became_active":"2024-09-13T14:39:32.489250+0000","last_became_peered":"2024-09-13T14:39:32.489250+0000","last_unstale":"2024-09-13T14:49:20.545834+0000","last_undegraded":"2024-09-13T14:49:20.545834+0000","last_fullsized":"2024-09-13T14:49:20.545834+0000","mapping_epoch":303,"log_start":"309'4200","ondisk_log_start":"309'4200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:43.471881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11204,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":372,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509312+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:49:19.509312+0000","last_peered":"2024-09-13T14:49:19.509312+0000","last_clean":"2024-09-13T14:49:19.509312+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:49:19.509312+0000","last_undegraded":"2024-09-13T14:49:19.509312+0000","last_fullsized":"2024-09-13T14:49:19.509312+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":584,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174337+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:49:21.174337+0000","last_peered":"2024-09-13T14:49:21.174337+0000","last_clean":"2024-09-13T14:49:21.174337+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:49:21.174337+0000","last_undegraded":"2024-09-13T14:49:21.174337+0000","last_fullsized":"2024-09-13T14:49:21.174337+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":767,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189437+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:49:20.189437+0000","last_peered":"2024-09-13T14:49:20.189437+0000","last_clean":"2024-09-13T14:49:20.189437+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:49:20.189437+0000","last_undegraded":"2024-09-13T14:49:20.189437+0000","last_fullsized":"2024-09-13T14:49:20.189437+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":619,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.177372+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:49:21.177372+0000","last_peered":"2024-09-13T14:49:21.177372+0000","last_clean":"2024-09-13T14:49:21.177372+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:49:21.177372+0000","last_undegraded":"2024-09-13T14:49:21.177372+0000","last_fullsized":"2024-09-13T14:49:21.177372+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"370'11690","reported_seq":11845,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174122+0000","last_change":"2024-09-13T14:39:32.315370+0000","last_active":"2024-09-13T14:49:21.174122+0000","last_peered":"2024-09-13T14:49:21.174122+0000","last_clean":"2024-09-13T14:49:21.174122+0000","last_became_active":"2024-09-13T14:39:32.315064+0000","last_became_peered":"2024-09-13T14:39:32.315064+0000","last_unstale":"2024-09-13T14:49:21.174122+0000","last_undegraded":"2024-09-13T14:49:21.174122+0000","last_fullsized":"2024-09-13T14:49:21.174122+0000","mapping_epoch":303,"log_start":"307'3100","ondisk_log_start":"307'3100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":8590,"log_dups_size":0,"ondisk_log_size":8590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:10.847300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11690,"num_write_kb":374080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511121+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:49:19.511121+0000","last_peered":"2024-09-13T14:49:19.511121+0000","last_clean":"2024-09-13T14:49:19.511121+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:49:19.511121+0000","last_undegraded":"2024-09-13T14:49:19.511121+0000","last_fullsized":"2024-09-13T14:49:19.511121+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":619,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545980+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:49:20.545980+0000","last_peered":"2024-09-13T14:49:20.545980+0000","last_clean":"2024-09-13T14:49:20.545980+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:49:20.545980+0000","last_undegraded":"2024-09-13T14:49:20.545980+0000","last_fullsized":"2024-09-13T14:49:20.545980+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":959,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174996+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:49:21.174996+0000","last_peered":"2024-09-13T14:49:21.174996+0000","last_clean":"2024-09-13T14:49:21.174996+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:49:21.174996+0000","last_undegraded":"2024-09-13T14:49:21.174996+0000","last_fullsized":"2024-09-13T14:49:21.174996+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":617,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.547921+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:49:20.547921+0000","last_peered":"2024-09-13T14:49:20.547921+0000","last_clean":"2024-09-13T14:49:20.547921+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:49:20.547921+0000","last_undegraded":"2024-09-13T14:49:20.547921+0000","last_fullsized":"2024-09-13T14:49:20.547921+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"370'11514","reported_seq":22898,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510842+0000","last_change":"2024-09-13T14:42:18.677420+0000","last_active":"2024-09-13T14:49:19.510842+0000","last_peered":"2024-09-13T14:49:19.510842+0000","last_clean":"2024-09-13T14:49:19.510842+0000","last_became_active":"2024-09-13T14:42:18.662131+0000","last_became_peered":"2024-09-13T14:42:18.662131+0000","last_unstale":"2024-09-13T14:49:19.510842+0000","last_undegraded":"2024-09-13T14:49:19.510842+0000","last_fullsized":"2024-09-13T14:49:19.510842+0000","mapping_epoch":332,"log_start":"307'2400","ondisk_log_start":"307'2400","created":303,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:39:31.301239+0000","last_clean_scrub_stamp":"2024-09-13T14:39:31.301239+0000","objects_scrubbed":0,"log_size":9114,"log_dups_size":0,"ondisk_log_size":9114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:10.278083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11514,"num_write_kb":368385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5662,"num_bytes_recovered":360316928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":471,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545317+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:49:20.545317+0000","last_peered":"2024-09-13T14:49:20.545317+0000","last_clean":"2024-09-13T14:49:20.545317+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:49:20.545317+0000","last_undegraded":"2024-09-13T14:49:20.545317+0000","last_fullsized":"2024-09-13T14:49:20.545317+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":654,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174670+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:49:21.174670+0000","last_peered":"2024-09-13T14:49:21.174670+0000","last_clean":"2024-09-13T14:49:21.174670+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:49:21.174670+0000","last_undegraded":"2024-09-13T14:49:21.174670+0000","last_fullsized":"2024-09-13T14:49:21.174670+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":791,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509458+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:49:19.509458+0000","last_peered":"2024-09-13T14:49:19.509458+0000","last_clean":"2024-09-13T14:49:19.509458+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:49:19.509458+0000","last_undegraded":"2024-09-13T14:49:19.509458+0000","last_fullsized":"2024-09-13T14:49:19.509458+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510110+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:49:19.510110+0000","last_peered":"2024-09-13T14:49:19.510110+0000","last_clean":"2024-09-13T14:49:19.510110+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:49:19.510110+0000","last_undegraded":"2024-09-13T14:49:19.510110+0000","last_fullsized":"2024-09-13T14:49:19.510110+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510565+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:49:19.510565+0000","last_peered":"2024-09-13T14:49:19.510565+0000","last_clean":"2024-09-13T14:49:19.510565+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:49:19.510565+0000","last_undegraded":"2024-09-13T14:49:19.510565+0000","last_fullsized":"2024-09-13T14:49:19.510565+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":82,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510222+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:49:19.510222+0000","last_peered":"2024-09-13T14:49:19.510222+0000","last_clean":"2024-09-13T14:49:19.510222+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:49:19.510222+0000","last_undegraded":"2024-09-13T14:49:19.510222+0000","last_fullsized":"2024-09-13T14:49:19.510222+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":571,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174647+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:49:21.174647+0000","last_peered":"2024-09-13T14:49:21.174647+0000","last_clean":"2024-09-13T14:49:21.174647+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:49:21.174647+0000","last_undegraded":"2024-09-13T14:49:21.174647+0000","last_fullsized":"2024-09-13T14:49:21.174647+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":372,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.189304+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:49:20.189304+0000","last_peered":"2024-09-13T14:49:20.189304+0000","last_clean":"2024-09-13T14:49:20.189304+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:49:20.189304+0000","last_undegraded":"2024-09-13T14:49:20.189304+0000","last_fullsized":"2024-09-13T14:49:20.189304+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":338,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511207+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:49:19.511207+0000","last_peered":"2024-09-13T14:49:19.511207+0000","last_clean":"2024-09-13T14:49:19.511207+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:49:19.511207+0000","last_undegraded":"2024-09-13T14:49:19.511207+0000","last_fullsized":"2024-09-13T14:49:19.511207+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":576,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.544914+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:49:20.544914+0000","last_peered":"2024-09-13T14:49:20.544914+0000","last_clean":"2024-09-13T14:49:20.544914+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:49:20.544914+0000","last_undegraded":"2024-09-13T14:49:20.544914+0000","last_fullsized":"2024-09-13T14:49:20.544914+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":627,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174352+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:49:21.174352+0000","last_peered":"2024-09-13T14:49:21.174352+0000","last_clean":"2024-09-13T14:49:21.174352+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:49:21.174352+0000","last_undegraded":"2024-09-13T14:49:21.174352+0000","last_fullsized":"2024-09-13T14:49:21.174352+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":632,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.547923+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:49:20.547923+0000","last_peered":"2024-09-13T14:49:20.547923+0000","last_clean":"2024-09-13T14:49:20.547923+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:49:20.547923+0000","last_undegraded":"2024-09-13T14:49:20.547923+0000","last_fullsized":"2024-09-13T14:49:20.547923+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":370,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509390+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:49:19.509390+0000","last_peered":"2024-09-13T14:49:19.509390+0000","last_clean":"2024-09-13T14:49:19.509390+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:49:19.509390+0000","last_undegraded":"2024-09-13T14:49:19.509390+0000","last_fullsized":"2024-09-13T14:49:19.509390+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":649,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.547961+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:49:20.547961+0000","last_peered":"2024-09-13T14:49:20.547961+0000","last_clean":"2024-09-13T14:49:20.547961+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:49:20.547961+0000","last_undegraded":"2024-09-13T14:49:20.547961+0000","last_fullsized":"2024-09-13T14:49:20.547961+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":620,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174020+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:49:21.174020+0000","last_peered":"2024-09-13T14:49:21.174020+0000","last_clean":"2024-09-13T14:49:21.174020+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:49:21.174020+0000","last_undegraded":"2024-09-13T14:49:21.174020+0000","last_fullsized":"2024-09-13T14:49:21.174020+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":596,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175338+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:49:21.175338+0000","last_peered":"2024-09-13T14:49:21.175338+0000","last_clean":"2024-09-13T14:49:21.175338+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:49:21.175338+0000","last_undegraded":"2024-09-13T14:49:21.175338+0000","last_fullsized":"2024-09-13T14:49:21.175338+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":645,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545320+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:49:20.545320+0000","last_peered":"2024-09-13T14:49:20.545320+0000","last_clean":"2024-09-13T14:49:20.545320+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:49:20.545320+0000","last_undegraded":"2024-09-13T14:49:20.545320+0000","last_fullsized":"2024-09-13T14:49:20.545320+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":71,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510171+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:49:19.510171+0000","last_peered":"2024-09-13T14:49:19.510171+0000","last_clean":"2024-09-13T14:49:19.510171+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:49:19.510171+0000","last_undegraded":"2024-09-13T14:49:19.510171+0000","last_fullsized":"2024-09-13T14:49:19.510171+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":542,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174610+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:49:21.174610+0000","last_peered":"2024-09-13T14:49:21.174610+0000","last_clean":"2024-09-13T14:49:21.174610+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:49:21.174610+0000","last_undegraded":"2024-09-13T14:49:21.174610+0000","last_fullsized":"2024-09-13T14:49:21.174610+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":623,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.175143+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:49:21.175143+0000","last_peered":"2024-09-13T14:49:21.175143+0000","last_clean":"2024-09-13T14:49:21.175143+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:49:21.175143+0000","last_undegraded":"2024-09-13T14:49:21.175143+0000","last_fullsized":"2024-09-13T14:49:21.175143+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510401+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:49:19.510401+0000","last_peered":"2024-09-13T14:49:19.510401+0000","last_clean":"2024-09-13T14:49:19.510401+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:49:19.510401+0000","last_undegraded":"2024-09-13T14:49:19.510401+0000","last_fullsized":"2024-09-13T14:49:19.510401+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":338,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510653+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:49:19.510653+0000","last_peered":"2024-09-13T14:49:19.510653+0000","last_clean":"2024-09-13T14:49:19.510653+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:49:19.510653+0000","last_undegraded":"2024-09-13T14:49:19.510653+0000","last_fullsized":"2024-09-13T14:49:19.510653+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":71,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.510073+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:49:19.510073+0000","last_peered":"2024-09-13T14:49:19.510073+0000","last_clean":"2024-09-13T14:49:19.510073+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:49:19.510073+0000","last_undegraded":"2024-09-13T14:49:19.510073+0000","last_fullsized":"2024-09-13T14:49:19.510073+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":304,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.511114+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:49:19.511114+0000","last_peered":"2024-09-13T14:49:19.511114+0000","last_clean":"2024-09-13T14:49:19.511114+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:49:19.511114+0000","last_undegraded":"2024-09-13T14:49:19.511114+0000","last_fullsized":"2024-09-13T14:49:19.511114+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":641,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545066+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:49:20.545066+0000","last_peered":"2024-09-13T14:49:20.545066+0000","last_clean":"2024-09-13T14:49:20.545066+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:49:20.545066+0000","last_undegraded":"2024-09-13T14:49:20.545066+0000","last_fullsized":"2024-09-13T14:49:20.545066+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":628,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:20.545741+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:49:20.545741+0000","last_peered":"2024-09-13T14:49:20.545741+0000","last_clean":"2024-09-13T14:49:20.545741+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:49:20.545741+0000","last_undegraded":"2024-09-13T14:49:20.545741+0000","last_fullsized":"2024-09-13T14:49:20.545741+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-09-13T14:49:19.509797+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:49:19.509797+0000","last_peered":"2024-09-13T14:49:19.509797+0000","last_clean":"2024-09-13T14:49:19.509797+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:49:19.509797+0000","last_undegraded":"2024-09-13T14:49:19.509797+0000","last_fullsized":"2024-09-13T14:49:19.509797+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":633,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-09-13T14:49:21.174245+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:49:21.174245+0000","last_peered":"2024-09-13T14:49:21.174245+0000","last_clean":"2024-09-13T14:49:21.174245+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:49:21.174245+0000","last_undegraded":"2024-09-13T14:49:21.174245+0000","last_fullsized":"2024-09-13T14:49:21.174245+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25776,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182678,"num_write_kb":5845633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33327,"num_bytes_recovered":1908867072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1042808832,"data_stored":1042808832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125578,"ondisk_log_size":125578,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":488689,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":79,"num_read_kb":204,"num_write":40,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397962,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896908,"kb_used_data":9216,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":92848244,"statfs":{"total":95995035648,"available":95076601856,"internally_reserved":0,"allocated":9437184,"data_stored":7421316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594190,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266560,"kb_used_data":359600,"kb_used_omap":1,"kb_used_meta":906942,"kb_avail":92478592,"statfs":{"total":95995035648,"available":94698078208,"internally_reserved":0,"allocated":368230400,"data_stored":366230214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":928708952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665644,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333748,"kb_used_data":5356,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93411404,"statfs":{"total":95995035648,"available":95653277696,"internally_reserved":0,"allocated":5484544,"data_stored":3484761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986646,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900492,"kb_used_data":4796,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92844660,"statfs":{"total":95995035648,"available":95072931840,"internally_reserved":0,"allocated":4911104,"data_stored":2899262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697206,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1140156,"kb_used_data":363948,"kb_used_omap":4,"kb_used_meta":776187,"kb_avail":92604996,"statfs":{"total":95995035648,"available":94827515904,"internally_reserved":0,"allocated":372682752,"data_stored":370683050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":794815563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860732,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":949188,"kb_used_data":310000,"kb_used_omap":8,"kb_used_meta":639159,"kb_avail":92795964,"statfs":{"total":95995035648,"available":95023067136,"internally_reserved":0,"allocated":317440000,"data_stored":315436716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":654499660},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020404,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":352692,"kb_used_data":8872,"kb_used_omap":9,"kb_used_meta":343798,"kb_avail":93392460,"statfs":{"total":95995035648,"available":95633879040,"internally_reserved":0,"allocated":9084928,"data_stored":7088699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":352049989},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187794,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":653432,"kb_used_data":8872,"kb_used_omap":7,"kb_used_meta":644536,"kb_avail":93091720,"statfs":{"total":95995035648,"available":95325921280,"internally_reserved":0,"allocated":9084928,"data_stored":7088965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":660005195},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":312279040,"data_stored":312279040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":367198208,"data_stored":367198208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":363331584,"data_stored":363331584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:49:48.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:49:48.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:48.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:48.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:48.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:48.540 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:48.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:48.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:48.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:48.854 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.878 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:48.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:48.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[103001]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24726/84 objects degraded (29435.714%); 0 B/s, 215 objects/s recovering 2024-09-13T14:49:48.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:49:48.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[103001]: osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[103001]: 1.1 deep-scrub starts 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[103001]: 1.1 deep-scrub ok 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[107556]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24726/84 objects degraded (29435.714%); 0 B/s, 215 objects/s recovering 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[107556]: osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[107556]: 1.1 deep-scrub starts 2024-09-13T14:49:48.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:48 smithi017 ceph-mon[107556]: 1.1 deep-scrub ok 2024-09-13T14:49:48.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:49.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:48 smithi189 ceph-mon[90992]: pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24726/84 objects degraded (29435.714%); 0 B/s, 215 objects/s recovering 2024-09-13T14:49:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/403672732' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:49:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:48 smithi189 ceph-mon[90992]: osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:49:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:48 smithi189 ceph-mon[90992]: 1.1 deep-scrub starts 2024-09-13T14:49:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:48 smithi189 ceph-mon[90992]: 1.1 deep-scrub ok 2024-09-13T14:49:49.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.273 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:49.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:49.332 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:49:49.333 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:49:49.333 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-13T14:49:49.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-13T14:49:49.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:49.417 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:49.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:49.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:49.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:49.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:49.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:50.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:50.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:50.132 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:50.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:50.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:50.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:50.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:50.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:50.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:49:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:49:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:50.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:50.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:50.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:50.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:50.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:50.720 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:50.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[103001]: pgmap v1244: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:50.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized) 2024-09-13T14:49:50.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:49:50.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[107556]: pgmap v1244: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:50.794 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized) 2024-09-13T14:49:50.794 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:50 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:49:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.796 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:50.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:50.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:50.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:50.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:50 smithi189 ceph-mon[90992]: pgmap v1244: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:50 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24726/84 objects degraded (29435.714%), 6 pgs degraded, 6 pgs undersized) 2024-09-13T14:49:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:50 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:49:51.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:51.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:51.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:51.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:51.652 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:51.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:51.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:51.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:51.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:51 smithi017 ceph-mon[103001]: 1.8 scrub starts 2024-09-13T14:49:51.862 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:51 smithi017 ceph-mon[103001]: 1.8 scrub ok 2024-09-13T14:49:51.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:51 smithi017 ceph-mon[107556]: 1.8 scrub starts 2024-09-13T14:49:51.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:51 smithi017 ceph-mon[107556]: 1.8 scrub ok 2024-09-13T14:49:51.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:51.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:52.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:51 smithi189 ceph-mon[90992]: 1.8 scrub starts 2024-09-13T14:49:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:51 smithi189 ceph-mon[90992]: 1.8 scrub ok 2024-09-13T14:49:52.037 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:52.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:52.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:52.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:52.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:52.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:52.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:52.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:52.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:52.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:52.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:52.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:52 smithi017 ceph-mon[107556]: pgmap v1245: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:52 smithi017 ceph-mon[103001]: pgmap v1245: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:52.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:52.821 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:52.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:52.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:52.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:52.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:52 smithi189 ceph-mon[90992]: pgmap v1245: 83 pgs: 83 active+clean; 4.4 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:53.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:53.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:53.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:53.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:53.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:53.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:53.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:53.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:53.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:53.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:53.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:53.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:53.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:54.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:54.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:54.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:54.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:54.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:54.431 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:54.454 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:54.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:54.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:54.754 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:54 smithi017 ceph-mon[103001]: pgmap v1246: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:54.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:54 smithi017 ceph-mon[107556]: pgmap v1246: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:54.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:54.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:54.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:54.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:54 smithi189 ceph-mon[90992]: pgmap v1246: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:55.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:55.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.182 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:55.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:55.425 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:55.432 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:49:55.433 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:49:55.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:55.543 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:49:55.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:55.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[103001]: 1.a scrub starts 2024-09-13T14:49:55.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[103001]: 1.a scrub ok 2024-09-13T14:49:55.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:55.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[107556]: 1.a scrub starts 2024-09-13T14:49:55.549 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:55 smithi017 ceph-mon[107556]: 1.a scrub ok 2024-09-13T14:49:55.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.587 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:55.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:55.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:55.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:55.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:55 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:55 smithi189 ceph-mon[90992]: 1.a scrub starts 2024-09-13T14:49:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:55 smithi189 ceph-mon[90992]: 1.a scrub ok 2024-09-13T14:49:56.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:56.087 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:56.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:56.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:56.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:56.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.467 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:56.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[103001]: pgmap v1247: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[103001]: osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[107556]: pgmap v1247: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:56.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:56 smithi017 ceph-mon[107556]: osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:49:56.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:56.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:56.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:56 smithi189 ceph-mon[90992]: pgmap v1247: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:56 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4260231424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:56 smithi189 ceph-mon[90992]: osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:49:57.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:57.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.140 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:57.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:57.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:57.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:57.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:57.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.639 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:49:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:49:57] "GET /metrics HTTP/1.1" 200 34314 "" "Prometheus/2.51.0" 2024-09-13T14:49:57.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:57.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:57.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:58.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:58.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:58.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:58.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:49:58.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.396 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:58.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: pgmap v1249: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3448546780' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: pgmap v1249: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3448546780' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.499 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' created 2024-09-13T14:49:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: pgmap v1249: 83 pgs: 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:49:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:49:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3448546780' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:49:58.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:58.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:58.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:49:58.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:58.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:59.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:59.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:59.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:59.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:59.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:59.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:59.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:59.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:59 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:49:59.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:49:59 smithi189 ceph-mon[90992]: osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:49:59.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:49:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:59 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:49:59.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:49:59 smithi017 ceph-mon[107556]: osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:49:59.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:59 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:49:59.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:49:59 smithi017 ceph-mon[103001]: osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:49:59.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:59.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:49:59.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:00.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:00.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:00.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:00.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:00.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.502 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:00.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:50:00.613 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:00.631 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:50:00.631 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:00.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:00.645 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:50:00.646 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1797,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":378,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238443,"num_in_osds":8,"osd_in_since":1726238425,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"unknown","count":13},{"state_name":"creating+peering","count":3}],"num_pgs":99,"num_pools":5,"num_objects":28,"data_bytes":4790221,"bytes_used":6301667328,"bytes_avail":761658617856,"bytes_total":767960285184,"unknown_pgs_ratio":0.13131313025951385,"inactive_pgs_ratio":0.030303031206130981},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ldrmqj","status":"up:active","gid":24539}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":28,"modified":"2024-09-13T14:49:13.295911+0000","services":{}},"progress_events":{"e755688a-ec8d-4647-9851-dddba6738f7e":{"message":"Upgrade to 19.3.0-4697-gd58f7306 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-13T14:50:00.649 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:50:00.649 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:00.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: pgmap v1251: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: application not enabled on pool 'unique_pool_4' 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[107556]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: pgmap v1251: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: application not enabled on pool 'unique_pool_4' 2024-09-13T14:50:00.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:00 smithi017 ceph-mon[103001]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:50:00.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:50:00.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:00.689 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:50:00.689 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:00.705 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:50:00.705 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:00.722 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:50:00.722 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:00.738 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:50:00.738 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:00.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: pgmap v1251: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: application not enabled on pool 'unique_pool_4' 2024-09-13T14:50:00.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:00 smithi189 ceph-mon[90992]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:50:00.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:00.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:00.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:00.913 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:01.140 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.162 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:01.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:01.285 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:01.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:01.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:01.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:01.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-13T14:50:01.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:01.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:01.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:01.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.670 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[103001]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:50:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3213740938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/626223328' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:50:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[107556]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:50:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3213740938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/626223328' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:50:01.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.917 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:01.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:01 smithi189 ceph-mon[90992]: osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:50:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3213740938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:02.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/626223328' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:50:02.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:02.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:02.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:02.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:02.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:02.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:02.393 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:02.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:02.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.641 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:02.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:02.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:02 smithi017 ceph-mon[103001]: pgmap v1254: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:02 smithi017 ceph-mon[107556]: pgmap v1254: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:02.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:02.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:02.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.913 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:02 smithi189 ceph-mon[90992]: pgmap v1254: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:03.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:03.027 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-09-13T14:50:03.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:50:03.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:03.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:03.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:03.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:03.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:03.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:03.990 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:03.995 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:04.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.027 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:04.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:04.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.278 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:04.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:04.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:04.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:04.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.559 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:04.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:04.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:04 smithi017 ceph-mon[107556]: pgmap v1255: 99 pgs: 3 creating+peering, 96 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:04.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:04 smithi017 ceph-mon[103001]: pgmap v1255: 99 pgs: 3 creating+peering, 96 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:04.909 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:04.934 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:04 smithi189 ceph-mon[90992]: pgmap v1255: 99 pgs: 3 creating+peering, 96 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:05.036 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:05.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:05.162 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:05.188 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:05.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:05.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:05.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:05.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:05.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:05.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:05.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:05.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:05.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:05.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:05.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:06.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:06.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:06.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.319 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.421 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.447 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:06.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:06.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:06.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:06 smithi017 ceph-mon[103001]: pgmap v1256: 99 pgs: 99 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:06 smithi017 ceph-mon[107556]: pgmap v1256: 99 pgs: 99 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:06.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:06.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:06.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:06.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:06 smithi189 ceph-mon[90992]: pgmap v1256: 99 pgs: 99 active+clean; 4.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:07.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:07.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:07.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:07.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:07.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:07.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:07.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:07.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:07.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:07.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:07] "GET /metrics HTTP/1.1" 200 34314 "" "Prometheus/2.51.0" 2024-09-13T14:50:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:07.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:07.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:07.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:07.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:08.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:08.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:08.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:08.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:08.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:08.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:08.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:08 smithi017 ceph-mon[103001]: pgmap v1257: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:08 smithi017 ceph-mon[107556]: pgmap v1257: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:08.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:08.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:08 smithi189 ceph-mon[90992]: pgmap v1257: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:09.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:09.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:09.152 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:09.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:09.284 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:09.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:09.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:09.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:09.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:09.525 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:50:09.528 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-13T14:50:09.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/860507853' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:09.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:09.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:09.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:09.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:09.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/860507853' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:09.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:09.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:09.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:09.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:09.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:09.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:09.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.857 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:09.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:09.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/860507853' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:50:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:10.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.252 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:50:10.252 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:50:10.252 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:50:10.252 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:50:10.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:10.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:10.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:10.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:10.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:10.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:10.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: pgmap v1258: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:10.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:10.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:50:10.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:10.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: pgmap v1258: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:50:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:10.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:50:10.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: pgmap v1258: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3288659959' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:11.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.341 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:11.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:11.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:11.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:11.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:11.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:11 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:50:11.813 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:11 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:50:11.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.870 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:11.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:11 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:50:12.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:12.489 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:12.563 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-13T14:50:12.565 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:50:12.566 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:50:12.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:12.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:12.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:12.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.737 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:50:12.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:50:12.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_339564 2024-09-13T14:50:12.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:50:12.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:50:12.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:12.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[103001]: pgmap v1260: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[107556]: pgmap v1260: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:12.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:12.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:12 smithi189 ceph-mon[90992]: pgmap v1260: 99 pgs: 99 active+clean; 4.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:50:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:13.012 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:13.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:13.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:13.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:13.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:13.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:13.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:13.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:13.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:13.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:13.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:13.797 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3859 3843 240.174 240.188 0.00419548 0.00415185 2024-09-13T14:50:13.823 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:13.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.993 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:14.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:14.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:50:14.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:14.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:14.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:14.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:14.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:14 smithi017 ceph-mon[103001]: pgmap v1261: 99 pgs: 99 active+clean; 16 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2024-09-13T14:50:14.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:14 smithi017 ceph-mon[107556]: pgmap v1261: 99 pgs: 99 active+clean; 16 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2024-09-13T14:50:14.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:14.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7237 7221 225.589 211.125 0.00507466 0.00442397 2024-09-13T14:50:14.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:14.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:14 smithi189 ceph-mon[90992]: pgmap v1261: 99 pgs: 99 active+clean; 16 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 18 op/s 2024-09-13T14:50:15.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:15.123 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:15.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:15.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:15.402 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:15.415 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:50:15.415 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:15.452 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:50:15.454 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:50:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:15.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:15.458 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:50:15.459 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:15.495 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:50:15.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:15.532 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:50:15.532 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:15.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:15.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:50:15.565 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:15.589 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:15.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:15.603 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:50:15.603 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:15.632 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:50:15.633 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:15.669 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:50:15.669 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:15.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:15.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:15.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 10103 10087 210.097 179.125 0.00704425 0.00475149 2024-09-13T14:50:15.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:15.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3730616603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3730616603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:15.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:15.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3730616603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:16.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:16.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:16.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:16.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:16.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:16.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:16.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:16.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 13322 13306 207.865 201.188 0.00544135 0.00480357 2024-09-13T14:50:16.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:16.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:16.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:16 smithi017 ceph-mon[103001]: pgmap v1262: 99 pgs: 99 active+clean; 136 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2024-09-13T14:50:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:16 smithi017 ceph-mon[103001]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:16 smithi017 ceph-mon[107556]: pgmap v1262: 99 pgs: 99 active+clean; 136 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2024-09-13T14:50:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:16 smithi017 ceph-mon[107556]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:16.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:16 smithi189 ceph-mon[90992]: pgmap v1262: 99 pgs: 99 active+clean; 136 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 13 MiB/s wr, 209 op/s 2024-09-13T14:50:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:16 smithi189 ceph-mon[90992]: from='client.36855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:17.075 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:17.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:17.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:17.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:17.350 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:17.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:17.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:17.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:17.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:17.663 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:17] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-09-13T14:50:17.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:17.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:17.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 16352 16336 204.164 189.375 0.0035687 0.00489167 2024-09-13T14:50:17.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:17.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:18.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:18.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:18.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:18.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:18.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:18.309 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:18.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:18.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.477 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:18.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:18.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:18.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:18.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:18.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 19014 18998 197.863 166.375 0.00738766 0.00504764 2024-09-13T14:50:18.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:18.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:18 smithi017 ceph-mon[107556]: pgmap v1263: 99 pgs: 99 active+clean; 298 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s 2024-09-13T14:50:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:18 smithi017 ceph-mon[103001]: pgmap v1263: 99 pgs: 99 active+clean; 298 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s 2024-09-13T14:50:18.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:19.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:18 smithi189 ceph-mon[90992]: pgmap v1263: 99 pgs: 99 active+clean; 298 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s wr, 469 op/s 2024-09-13T14:50:19.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:19.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:19.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:19.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:19.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:19.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:19.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:19.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:19.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 21227 21211 189.343 138.312 0.0220532 0.00527384 2024-09-13T14:50:19.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:19.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:20.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:20.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:20.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:20.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:20.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:20.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:20.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:20.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 23233 23217 181.346 125.375 0.00479036 0.00550887 2024-09-13T14:50:20.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:20 smithi017 ceph-mon[103001]: pgmap v1264: 99 pgs: 99 active+clean; 916 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:20 smithi017 ceph-mon[107556]: pgmap v1264: 99 pgs: 99 active+clean; 916 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:20 smithi189 ceph-mon[90992]: pgmap v1264: 99 pgs: 99 active+clean; 916 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:21.007 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.042 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:21.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:21.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:21.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:21.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:21.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.565 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:21.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:21.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:21.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:21.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 25294 25278 175.506 128.812 0.00549657 0.00569187 2024-09-13T14:50:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:21.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:21.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:21.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:22.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:22.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:22.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:22.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:22.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:22.352 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:22.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:22.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:22.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:22.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:22.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:22.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:22.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 27590 27574 172.304 143.5 0.0062139 0.00579813 2024-09-13T14:50:22.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:22.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:22.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:22 smithi017 ceph-mon[103001]: pgmap v1265: 99 pgs: 99 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:22 smithi017 ceph-mon[107556]: pgmap v1265: 99 pgs: 99 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:22 smithi189 ceph-mon[90992]: pgmap v1265: 99 pgs: 99 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:23.045 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:23.078 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:23.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:23.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:23.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:23.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:23.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:23.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:23.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:23.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:23.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:23.744 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:23.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 29677 29661 168.498 130.438 0.0103476 0.00592916 2024-09-13T14:50:23.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:23.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:23.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:24.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:24.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:24.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:24.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:24.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:24.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.648 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:24.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:24.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:24.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:24.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 31148 31132 162.115 91.9375 0.0136436 0.00616314 2024-09-13T14:50:24.839 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1262,"stamp":"2024-09-13T14:50:15.312962+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":142284749,"num_objects":2126,"num_object_clones":0,"num_object_copies":6378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2126,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":3005,"num_write_kb":146895,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4230,"ondisk_log_size":4230,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6393832,"kb_used_data":992152,"kb_used_omap":44,"kb_used_meta":5401555,"kb_avail":743567384,"statfs":{"total":767960285184,"available":761413001216,"internally_reserved":0,"allocated":1015963648,"data_stored":999728317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5531192567},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":137494528,"num_objects":2098,"num_object_clones":0,"num_object_copies":6294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2098,"num_write_kb":134272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002915"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":644,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420848+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:50:10.420848+0000","last_peered":"2024-09-13T14:50:10.420848+0000","last_clean":"2024-09-13T14:50:10.420848+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:50:10.420848+0000","last_undegraded":"2024-09-13T14:50:10.420848+0000","last_fullsized":"2024-09-13T14:50:10.420848+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":377,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509453+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:50:00.509453+0000","last_peered":"2024-09-13T14:50:00.509453+0000","last_clean":"2024-09-13T14:50:00.509453+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:50:00.509453+0000","last_undegraded":"2024-09-13T14:50:00.509453+0000","last_fullsized":"2024-09-13T14:50:00.509453+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":306,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508753+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:50:00.508753+0000","last_peered":"2024-09-13T14:50:00.508753+0000","last_clean":"2024-09-13T14:50:00.508753+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:50:00.508753+0000","last_undegraded":"2024-09-13T14:50:00.508753+0000","last_fullsized":"2024-09-13T14:50:00.508753+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":629,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495070+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:50:10.495070+0000","last_peered":"2024-09-13T14:50:10.495070+0000","last_clean":"2024-09-13T14:50:10.495070+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:50:10.495070+0000","last_undegraded":"2024-09-13T14:50:10.495070+0000","last_fullsized":"2024-09-13T14:50:10.495070+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":629,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495360+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:50:10.495360+0000","last_peered":"2024-09-13T14:50:10.495360+0000","last_clean":"2024-09-13T14:50:10.495360+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:50:10.495360+0000","last_undegraded":"2024-09-13T14:50:10.495360+0000","last_fullsized":"2024-09-13T14:50:10.495360+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":346,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508319+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:50:00.508319+0000","last_peered":"2024-09-13T14:50:00.508319+0000","last_clean":"2024-09-13T14:50:00.508319+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:50:00.508319+0000","last_undegraded":"2024-09-13T14:50:00.508319+0000","last_fullsized":"2024-09-13T14:50:00.508319+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":643,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420984+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:50:10.420984+0000","last_peered":"2024-09-13T14:50:10.420984+0000","last_clean":"2024-09-13T14:50:10.420984+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:50:10.420984+0000","last_undegraded":"2024-09-13T14:50:10.420984+0000","last_fullsized":"2024-09-13T14:50:10.420984+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":368,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033791+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:50:01.033791+0000","last_peered":"2024-09-13T14:50:01.033791+0000","last_clean":"2024-09-13T14:50:01.033791+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:50:01.033791+0000","last_undegraded":"2024-09-13T14:50:01.033791+0000","last_fullsized":"2024-09-13T14:50:01.033791+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"379'325","reported_seq":342,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.118574+0000","last_change":"2024-09-13T14:49:59.509935+0000","last_active":"2024-09-13T14:50:14.118574+0000","last_peered":"2024-09-13T14:50:14.118574+0000","last_clean":"2024-09-13T14:50:14.118574+0000","last_became_active":"2024-09-13T14:49:59.509619+0000","last_became_peered":"2024-09-13T14:49:59.509619+0000","last_unstale":"2024-09-13T14:50:14.118574+0000","last_undegraded":"2024-09-13T14:50:14.118574+0000","last_fullsized":"2024-09-13T14:50:14.118574+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":325,"log_dups_size":0,"ondisk_log_size":325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:39:57.083056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":325,"num_write_kb":20800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":372,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509326+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:50:00.509326+0000","last_peered":"2024-09-13T14:50:00.509326+0000","last_clean":"2024-09-13T14:50:00.509326+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:50:00.509326+0000","last_undegraded":"2024-09-13T14:50:00.509326+0000","last_fullsized":"2024-09-13T14:50:00.509326+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":291,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.507877+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:50:00.507877+0000","last_peered":"2024-09-13T14:50:00.507877+0000","last_clean":"2024-09-13T14:50:00.507877+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:50:00.507877+0000","last_undegraded":"2024-09-13T14:50:00.507877+0000","last_fullsized":"2024-09-13T14:50:00.507877+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"379'491","reported_seq":508,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:15.137032+0000","last_change":"2024-09-13T14:49:59.755413+0000","last_active":"2024-09-13T14:50:15.137032+0000","last_peered":"2024-09-13T14:50:15.137032+0000","last_clean":"2024-09-13T14:50:15.137032+0000","last_became_active":"2024-09-13T14:49:59.755054+0000","last_became_peered":"2024-09-13T14:49:59.755054+0000","last_unstale":"2024-09-13T14:50:15.137032+0000","last_undegraded":"2024-09-13T14:50:15.137032+0000","last_fullsized":"2024-09-13T14:50:15.137032+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":491,"log_dups_size":0,"ondisk_log_size":491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:24.559415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32178176,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":491,"num_write_kb":31424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":390,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508862+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:50:00.508862+0000","last_peered":"2024-09-13T14:50:00.508862+0000","last_clean":"2024-09-13T14:50:00.508862+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:50:00.508862+0000","last_undegraded":"2024-09-13T14:50:00.508862+0000","last_fullsized":"2024-09-13T14:50:00.508862+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033449+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:50:01.033449+0000","last_peered":"2024-09-13T14:50:01.033449+0000","last_clean":"2024-09-13T14:50:01.033449+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:50:01.033449+0000","last_undegraded":"2024-09-13T14:50:01.033449+0000","last_fullsized":"2024-09-13T14:50:01.033449+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"379'38","reported_seq":49,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:12.974620+0000","last_change":"2024-09-13T14:49:59.509888+0000","last_active":"2024-09-13T14:50:12.974620+0000","last_peered":"2024-09-13T14:50:12.974620+0000","last_clean":"2024-09-13T14:50:12.974620+0000","last_became_active":"2024-09-13T14:49:59.509562+0000","last_became_peered":"2024-09-13T14:49:59.509562+0000","last_unstale":"2024-09-13T14:50:12.974620+0000","last_undegraded":"2024-09-13T14:50:12.974620+0000","last_fullsized":"2024-09-13T14:50:12.974620+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":38,"log_dups_size":0,"ondisk_log_size":38,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:53.170631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2490368,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":38,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":344,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509487+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:50:00.509487+0000","last_peered":"2024-09-13T14:50:00.509487+0000","last_clean":"2024-09-13T14:50:00.509487+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:50:00.509487+0000","last_undegraded":"2024-09-13T14:50:00.509487+0000","last_fullsized":"2024-09-13T14:50:00.509487+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":359,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033630+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:50:01.033630+0000","last_peered":"2024-09-13T14:50:01.033630+0000","last_clean":"2024-09-13T14:50:01.033630+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:50:01.033630+0000","last_undegraded":"2024-09-13T14:50:01.033630+0000","last_fullsized":"2024-09-13T14:50:01.033630+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"379'50","reported_seq":61,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:12.977215+0000","last_change":"2024-09-13T14:50:00.002234+0000","last_active":"2024-09-13T14:50:12.977215+0000","last_peered":"2024-09-13T14:50:12.977215+0000","last_clean":"2024-09-13T14:50:12.977215+0000","last_became_active":"2024-09-13T14:50:00.001900+0000","last_became_peered":"2024-09-13T14:50:00.001900+0000","last_unstale":"2024-09-13T14:50:12.977215+0000","last_undegraded":"2024-09-13T14:50:12.977215+0000","last_fullsized":"2024-09-13T14:50:12.977215+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:03.241074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":50,"num_write_kb":3200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508228+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:50:00.508228+0000","last_peered":"2024-09-13T14:50:00.508228+0000","last_clean":"2024-09-13T14:50:00.508228+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:50:00.508228+0000","last_undegraded":"2024-09-13T14:50:00.508228+0000","last_fullsized":"2024-09-13T14:50:00.508228+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":655,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494456+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:50:10.494456+0000","last_peered":"2024-09-13T14:50:10.494456+0000","last_clean":"2024-09-13T14:50:10.494456+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:50:10.494456+0000","last_undegraded":"2024-09-13T14:50:10.494456+0000","last_fullsized":"2024-09-13T14:50:10.494456+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420476+0000","last_change":"2024-09-13T14:49:59.511081+0000","last_active":"2024-09-13T14:50:10.420476+0000","last_peered":"2024-09-13T14:50:10.420476+0000","last_clean":"2024-09-13T14:50:10.420476+0000","last_became_active":"2024-09-13T14:49:59.510738+0000","last_became_peered":"2024-09-13T14:49:59.510738+0000","last_unstale":"2024-09-13T14:50:10.420476+0000","last_undegraded":"2024-09-13T14:50:10.420476+0000","last_fullsized":"2024-09-13T14:50:10.420476+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:53.029264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":490,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508093+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:50:00.508093+0000","last_peered":"2024-09-13T14:50:00.508093+0000","last_clean":"2024-09-13T14:50:00.508093+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:50:00.508093+0000","last_undegraded":"2024-09-13T14:50:00.508093+0000","last_fullsized":"2024-09-13T14:50:00.508093+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":653,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420580+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:50:10.420580+0000","last_peered":"2024-09-13T14:50:10.420580+0000","last_clean":"2024-09-13T14:50:10.420580+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:50:10.420580+0000","last_undegraded":"2024-09-13T14:50:10.420580+0000","last_fullsized":"2024-09-13T14:50:10.420580+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.034285+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:50:01.034285+0000","last_peered":"2024-09-13T14:50:01.034285+0000","last_clean":"2024-09-13T14:50:01.034285+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:50:01.034285+0000","last_undegraded":"2024-09-13T14:50:01.034285+0000","last_fullsized":"2024-09-13T14:50:01.034285+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"379'498","reported_seq":515,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:15.146822+0000","last_change":"2024-09-13T14:50:00.002352+0000","last_active":"2024-09-13T14:50:15.146822+0000","last_peered":"2024-09-13T14:50:15.146822+0000","last_clean":"2024-09-13T14:50:15.146822+0000","last_became_active":"2024-09-13T14:50:00.002036+0000","last_became_peered":"2024-09-13T14:50:00.002036+0000","last_unstale":"2024-09-13T14:50:15.146822+0000","last_undegraded":"2024-09-13T14:50:15.146822+0000","last_fullsized":"2024-09-13T14:50:15.146822+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":498,"log_dups_size":0,"ondisk_log_size":498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:29:00.760558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":498,"num_write_kb":31872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":204,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.507911+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:50:00.507911+0000","last_peered":"2024-09-13T14:50:00.507911+0000","last_clean":"2024-09-13T14:50:00.507911+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:50:00.507911+0000","last_undegraded":"2024-09-13T14:50:00.507911+0000","last_fullsized":"2024-09-13T14:50:00.507911+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":380,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033492+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:50:01.033492+0000","last_peered":"2024-09-13T14:50:01.033492+0000","last_clean":"2024-09-13T14:50:01.033492+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:50:01.033492+0000","last_undegraded":"2024-09-13T14:50:01.033492+0000","last_fullsized":"2024-09-13T14:50:01.033492+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.034182+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:50:01.034182+0000","last_peered":"2024-09-13T14:50:01.034182+0000","last_clean":"2024-09-13T14:50:01.034182+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:50:01.034182+0000","last_undegraded":"2024-09-13T14:50:01.034182+0000","last_fullsized":"2024-09-13T14:50:01.034182+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":11,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508701+0000","last_change":"2024-09-13T14:49:59.511648+0000","last_active":"2024-09-13T14:50:00.508701+0000","last_peered":"2024-09-13T14:50:00.508701+0000","last_clean":"2024-09-13T14:50:00.508701+0000","last_became_active":"2024-09-13T14:49:59.511315+0000","last_became_peered":"2024-09-13T14:49:59.511315+0000","last_unstale":"2024-09-13T14:50:00.508701+0000","last_undegraded":"2024-09-13T14:50:00.508701+0000","last_fullsized":"2024-09-13T14:50:00.508701+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:43.509125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033230+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:50:01.033230+0000","last_peered":"2024-09-13T14:50:01.033230+0000","last_clean":"2024-09-13T14:50:01.033230+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:50:01.033230+0000","last_undegraded":"2024-09-13T14:50:01.033230+0000","last_fullsized":"2024-09-13T14:50:01.033230+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.034650+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:50:01.034650+0000","last_peered":"2024-09-13T14:50:01.034650+0000","last_clean":"2024-09-13T14:50:01.034650+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:50:01.034650+0000","last_undegraded":"2024-09-13T14:50:01.034650+0000","last_fullsized":"2024-09-13T14:50:01.034650+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420367+0000","last_change":"2024-09-13T14:50:00.002592+0000","last_active":"2024-09-13T14:50:10.420367+0000","last_peered":"2024-09-13T14:50:10.420367+0000","last_clean":"2024-09-13T14:50:10.420367+0000","last_became_active":"2024-09-13T14:50:00.002051+0000","last_became_peered":"2024-09-13T14:50:00.002051+0000","last_unstale":"2024-09-13T14:50:10.420367+0000","last_undegraded":"2024-09-13T14:50:10.420367+0000","last_fullsized":"2024-09-13T14:50:10.420367+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:28:44.776933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":197,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508333+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:50:00.508333+0000","last_peered":"2024-09-13T14:50:00.508333+0000","last_clean":"2024-09-13T14:50:00.508333+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:50:00.508333+0000","last_undegraded":"2024-09-13T14:50:00.508333+0000","last_fullsized":"2024-09-13T14:50:00.508333+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":646,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420487+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:50:10.420487+0000","last_peered":"2024-09-13T14:50:10.420487+0000","last_clean":"2024-09-13T14:50:10.420487+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:50:10.420487+0000","last_undegraded":"2024-09-13T14:50:10.420487+0000","last_fullsized":"2024-09-13T14:50:10.420487+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":607,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527419+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:50:09.527419+0000","last_peered":"2024-09-13T14:50:09.527419+0000","last_clean":"2024-09-13T14:50:09.527419+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:50:09.527419+0000","last_undegraded":"2024-09-13T14:50:09.527419+0000","last_fullsized":"2024-09-13T14:50:09.527419+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420325+0000","last_change":"2024-09-13T14:49:59.509238+0000","last_active":"2024-09-13T14:50:10.420325+0000","last_peered":"2024-09-13T14:50:10.420325+0000","last_clean":"2024-09-13T14:50:10.420325+0000","last_became_active":"2024-09-13T14:49:59.508811+0000","last_became_peered":"2024-09-13T14:49:59.508811+0000","last_unstale":"2024-09-13T14:50:10.420325+0000","last_undegraded":"2024-09-13T14:50:10.420325+0000","last_fullsized":"2024-09-13T14:50:10.420325+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:43.422288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":481,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.507969+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:50:00.507969+0000","last_peered":"2024-09-13T14:50:00.507969+0000","last_clean":"2024-09-13T14:50:00.507969+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:50:00.507969+0000","last_undegraded":"2024-09-13T14:50:00.507969+0000","last_fullsized":"2024-09-13T14:50:00.507969+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":903,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508383+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:50:00.508383+0000","last_peered":"2024-09-13T14:50:00.508383+0000","last_clean":"2024-09-13T14:50:00.508383+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:50:00.508383+0000","last_undegraded":"2024-09-13T14:50:00.508383+0000","last_fullsized":"2024-09-13T14:50:00.508383+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":599,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420465+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:50:10.420465+0000","last_peered":"2024-09-13T14:50:10.420465+0000","last_clean":"2024-09-13T14:50:10.420465+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:50:10.420465+0000","last_undegraded":"2024-09-13T14:50:10.420465+0000","last_fullsized":"2024-09-13T14:50:10.420465+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":345,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033705+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:50:01.033705+0000","last_peered":"2024-09-13T14:50:01.033705+0000","last_clean":"2024-09-13T14:50:01.033705+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:50:01.033705+0000","last_undegraded":"2024-09-13T14:50:01.033705+0000","last_fullsized":"2024-09-13T14:50:01.033705+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"379'292","reported_seq":309,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.108565+0000","last_change":"2024-09-13T14:49:59.507963+0000","last_active":"2024-09-13T14:50:14.108565+0000","last_peered":"2024-09-13T14:50:14.108565+0000","last_clean":"2024-09-13T14:50:14.108565+0000","last_became_active":"2024-09-13T14:49:59.507611+0000","last_became_peered":"2024-09-13T14:49:59.507611+0000","last_unstale":"2024-09-13T14:50:14.108565+0000","last_undegraded":"2024-09-13T14:50:14.108565+0000","last_fullsized":"2024-09-13T14:50:14.108565+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":292,"log_dups_size":0,"ondisk_log_size":292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:32.773088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":292,"num_write_kb":18688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":453,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527140+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:50:09.527140+0000","last_peered":"2024-09-13T14:50:09.527140+0000","last_clean":"2024-09-13T14:50:09.527140+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:50:09.527140+0000","last_undegraded":"2024-09-13T14:50:09.527140+0000","last_fullsized":"2024-09-13T14:50:09.527140+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":769,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.507907+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:50:00.507907+0000","last_peered":"2024-09-13T14:50:00.507907+0000","last_clean":"2024-09-13T14:50:00.507907+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:50:00.507907+0000","last_undegraded":"2024-09-13T14:50:00.507907+0000","last_fullsized":"2024-09-13T14:50:00.507907+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":648,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495447+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:50:10.495447+0000","last_peered":"2024-09-13T14:50:10.495447+0000","last_clean":"2024-09-13T14:50:10.495447+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:50:10.495447+0000","last_undegraded":"2024-09-13T14:50:10.495447+0000","last_fullsized":"2024-09-13T14:50:10.495447+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":653,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495586+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:50:10.495586+0000","last_peered":"2024-09-13T14:50:10.495586+0000","last_clean":"2024-09-13T14:50:10.495586+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:50:10.495586+0000","last_undegraded":"2024-09-13T14:50:10.495586+0000","last_fullsized":"2024-09-13T14:50:10.495586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"379'46","reported_seq":57,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:12.975988+0000","last_change":"2024-09-13T14:49:59.509683+0000","last_active":"2024-09-13T14:50:12.975988+0000","last_peered":"2024-09-13T14:50:12.975988+0000","last_clean":"2024-09-13T14:50:12.975988+0000","last_became_active":"2024-09-13T14:49:59.509390+0000","last_became_peered":"2024-09-13T14:49:59.509390+0000","last_unstale":"2024-09-13T14:50:12.975988+0000","last_undegraded":"2024-09-13T14:50:12.975988+0000","last_fullsized":"2024-09-13T14:50:12.975988+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:08.503709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3014656,"num_objects":46,"num_object_clones":0,"num_object_copies":138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46,"num_write_kb":2944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":510,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508440+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:50:00.508440+0000","last_peered":"2024-09-13T14:50:00.508440+0000","last_clean":"2024-09-13T14:50:00.508440+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:50:00.508440+0000","last_undegraded":"2024-09-13T14:50:00.508440+0000","last_fullsized":"2024-09-13T14:50:00.508440+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":617,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420615+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:50:10.420615+0000","last_peered":"2024-09-13T14:50:10.420615+0000","last_clean":"2024-09-13T14:50:10.420615+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:50:10.420615+0000","last_undegraded":"2024-09-13T14:50:10.420615+0000","last_fullsized":"2024-09-13T14:50:10.420615+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":585,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.526616+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:50:09.526616+0000","last_peered":"2024-09-13T14:50:09.526616+0000","last_clean":"2024-09-13T14:50:09.526616+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:50:09.526616+0000","last_undegraded":"2024-09-13T14:50:09.526616+0000","last_fullsized":"2024-09-13T14:50:09.526616+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":128,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509138+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:50:00.509138+0000","last_peered":"2024-09-13T14:50:00.509138+0000","last_clean":"2024-09-13T14:50:00.509138+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:50:00.509138+0000","last_undegraded":"2024-09-13T14:50:00.509138+0000","last_fullsized":"2024-09-13T14:50:00.509138+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"379'304","reported_seq":321,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.116076+0000","last_change":"2024-09-13T14:49:59.510827+0000","last_active":"2024-09-13T14:50:14.116076+0000","last_peered":"2024-09-13T14:50:14.116076+0000","last_clean":"2024-09-13T14:50:14.116076+0000","last_became_active":"2024-09-13T14:49:59.509711+0000","last_became_peered":"2024-09-13T14:49:59.509711+0000","last_unstale":"2024-09-13T14:50:14.116076+0000","last_undegraded":"2024-09-13T14:50:14.116076+0000","last_fullsized":"2024-09-13T14:50:14.116076+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":304,"log_dups_size":0,"ondisk_log_size":304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:17.400887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19922944,"num_objects":304,"num_object_clones":0,"num_object_copies":912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":304,"num_write_kb":19456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":197,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508633+0000","last_change":"2024-09-13T14:49:47.727307+0000","last_active":"2024-09-13T14:50:00.508633+0000","last_peered":"2024-09-13T14:50:00.508633+0000","last_clean":"2024-09-13T14:50:00.508633+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:50:00.508633+0000","last_undegraded":"2024-09-13T14:50:00.508633+0000","last_fullsized":"2024-09-13T14:50:00.508633+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:30:33.609834+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":994,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508426+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:50:00.508426+0000","last_peered":"2024-09-13T14:50:00.508426+0000","last_clean":"2024-09-13T14:50:00.508426+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:50:00.508426+0000","last_undegraded":"2024-09-13T14:50:00.508426+0000","last_fullsized":"2024-09-13T14:50:00.508426+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033905+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:50:01.033905+0000","last_peered":"2024-09-13T14:50:01.033905+0000","last_clean":"2024-09-13T14:50:01.033905+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:50:01.033905+0000","last_undegraded":"2024-09-13T14:50:01.033905+0000","last_fullsized":"2024-09-13T14:50:01.033905+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":606,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.526863+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:50:09.526863+0000","last_peered":"2024-09-13T14:50:09.526863+0000","last_clean":"2024-09-13T14:50:09.526863+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:50:09.526863+0000","last_undegraded":"2024-09-13T14:50:09.526863+0000","last_fullsized":"2024-09-13T14:50:09.526863+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"379'54","reported_seq":64,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:12.971334+0000","last_change":"2024-09-13T14:49:59.507705+0000","last_active":"2024-09-13T14:50:12.971334+0000","last_peered":"2024-09-13T14:50:12.971334+0000","last_clean":"2024-09-13T14:50:12.971334+0000","last_became_active":"2024-09-13T14:49:59.507374+0000","last_became_peered":"2024-09-13T14:49:59.507374+0000","last_unstale":"2024-09-13T14:50:12.971334+0000","last_undegraded":"2024-09-13T14:50:12.971334+0000","last_fullsized":"2024-09-13T14:50:12.971334+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:22.009750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":54,"num_write_kb":3456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":378,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033381+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:50:01.033381+0000","last_peered":"2024-09-13T14:50:01.033381+0000","last_clean":"2024-09-13T14:50:01.033381+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:50:01.033381+0000","last_undegraded":"2024-09-13T14:50:01.033381+0000","last_fullsized":"2024-09-13T14:50:01.033381+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":324,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508570+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:50:00.508570+0000","last_peered":"2024-09-13T14:50:00.508570+0000","last_clean":"2024-09-13T14:50:00.508570+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:50:00.508570+0000","last_undegraded":"2024-09-13T14:50:00.508570+0000","last_fullsized":"2024-09-13T14:50:00.508570+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1185,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527077+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:50:09.527077+0000","last_peered":"2024-09-13T14:50:09.527077+0000","last_clean":"2024-09-13T14:50:09.527077+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:50:09.527077+0000","last_undegraded":"2024-09-13T14:50:09.527077+0000","last_fullsized":"2024-09-13T14:50:09.527077+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":587,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527065+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:50:09.527065+0000","last_peered":"2024-09-13T14:50:09.527065+0000","last_clean":"2024-09-13T14:50:09.527065+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:50:09.527065+0000","last_undegraded":"2024-09-13T14:50:09.527065+0000","last_fullsized":"2024-09-13T14:50:09.527065+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":11,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508569+0000","last_change":"2024-09-13T14:49:59.754749+0000","last_active":"2024-09-13T14:50:00.508569+0000","last_peered":"2024-09-13T14:50:00.508569+0000","last_clean":"2024-09-13T14:50:00.508569+0000","last_became_active":"2024-09-13T14:49:59.754398+0000","last_became_peered":"2024-09-13T14:49:59.754398+0000","last_unstale":"2024-09-13T14:50:00.508569+0000","last_undegraded":"2024-09-13T14:50:00.508569+0000","last_fullsized":"2024-09-13T14:50:00.508569+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:08.387473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":378,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.034298+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:50:01.034298+0000","last_peered":"2024-09-13T14:50:01.034298+0000","last_clean":"2024-09-13T14:50:01.034298+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:50:01.034298+0000","last_undegraded":"2024-09-13T14:50:01.034298+0000","last_fullsized":"2024-09-13T14:50:01.034298+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":596,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.526979+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:50:09.526979+0000","last_peered":"2024-09-13T14:50:09.526979+0000","last_clean":"2024-09-13T14:50:09.526979+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:50:09.526979+0000","last_undegraded":"2024-09-13T14:50:09.526979+0000","last_fullsized":"2024-09-13T14:50:09.526979+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":773,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509534+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:50:00.509534+0000","last_peered":"2024-09-13T14:50:00.509534+0000","last_clean":"2024-09-13T14:50:00.509534+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:50:00.509534+0000","last_undegraded":"2024-09-13T14:50:00.509534+0000","last_fullsized":"2024-09-13T14:50:00.509534+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":631,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494825+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:50:10.494825+0000","last_peered":"2024-09-13T14:50:10.494825+0000","last_clean":"2024-09-13T14:50:10.494825+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:50:10.494825+0000","last_undegraded":"2024-09-13T14:50:10.494825+0000","last_fullsized":"2024-09-13T14:50:10.494825+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":11,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509304+0000","last_change":"2024-09-13T14:49:59.511444+0000","last_active":"2024-09-13T14:50:00.509304+0000","last_peered":"2024-09-13T14:50:00.509304+0000","last_clean":"2024-09-13T14:50:00.509304+0000","last_became_active":"2024-09-13T14:49:59.511154+0000","last_became_peered":"2024-09-13T14:49:59.511154+0000","last_unstale":"2024-09-13T14:50:00.509304+0000","last_undegraded":"2024-09-13T14:50:00.509304+0000","last_fullsized":"2024-09-13T14:50:00.509304+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:18.382270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033982+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:50:01.033982+0000","last_peered":"2024-09-13T14:50:01.033982+0000","last_clean":"2024-09-13T14:50:01.033982+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:50:01.033982+0000","last_undegraded":"2024-09-13T14:50:01.033982+0000","last_fullsized":"2024-09-13T14:50:01.033982+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":632,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.421074+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:50:10.421074+0000","last_peered":"2024-09-13T14:50:10.421074+0000","last_clean":"2024-09-13T14:50:10.421074+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:50:10.421074+0000","last_undegraded":"2024-09-13T14:50:10.421074+0000","last_fullsized":"2024-09-13T14:50:10.421074+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":972,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527273+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:50:09.527273+0000","last_peered":"2024-09-13T14:50:09.527273+0000","last_clean":"2024-09-13T14:50:09.527273+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:50:09.527273+0000","last_undegraded":"2024-09-13T14:50:09.527273+0000","last_fullsized":"2024-09-13T14:50:09.527273+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":630,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420191+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:50:10.420191+0000","last_peered":"2024-09-13T14:50:10.420191+0000","last_clean":"2024-09-13T14:50:10.420191+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:50:10.420191+0000","last_undegraded":"2024-09-13T14:50:10.420191+0000","last_fullsized":"2024-09-13T14:50:10.420191+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":17,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420681+0000","last_change":"2024-09-13T14:49:59.755421+0000","last_active":"2024-09-13T14:50:10.420681+0000","last_peered":"2024-09-13T14:50:10.420681+0000","last_clean":"2024-09-13T14:50:10.420681+0000","last_became_active":"2024-09-13T14:49:59.755094+0000","last_became_peered":"2024-09-13T14:49:59.755094+0000","last_unstale":"2024-09-13T14:50:10.420681+0000","last_undegraded":"2024-09-13T14:50:10.420681+0000","last_fullsized":"2024-09-13T14:50:10.420681+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:27.511205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":483,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420634+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:50:10.420634+0000","last_peered":"2024-09-13T14:50:10.420634+0000","last_clean":"2024-09-13T14:50:10.420634+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:50:10.420634+0000","last_undegraded":"2024-09-13T14:50:10.420634+0000","last_fullsized":"2024-09-13T14:50:10.420634+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":667,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495159+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:50:10.495159+0000","last_peered":"2024-09-13T14:50:10.495159+0000","last_clean":"2024-09-13T14:50:10.495159+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:50:10.495159+0000","last_undegraded":"2024-09-13T14:50:10.495159+0000","last_fullsized":"2024-09-13T14:50:10.495159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":797,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033974+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:50:01.033974+0000","last_peered":"2024-09-13T14:50:01.033974+0000","last_clean":"2024-09-13T14:50:01.033974+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:50:01.033974+0000","last_undegraded":"2024-09-13T14:50:01.033974+0000","last_fullsized":"2024-09-13T14:50:01.033974+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033312+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:50:01.033312+0000","last_peered":"2024-09-13T14:50:01.033312+0000","last_clean":"2024-09-13T14:50:01.033312+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:50:01.033312+0000","last_undegraded":"2024-09-13T14:50:01.033312+0000","last_fullsized":"2024-09-13T14:50:01.033312+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033978+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:50:01.033978+0000","last_peered":"2024-09-13T14:50:01.033978+0000","last_clean":"2024-09-13T14:50:01.033978+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:50:01.033978+0000","last_undegraded":"2024-09-13T14:50:01.033978+0000","last_fullsized":"2024-09-13T14:50:01.033978+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":88,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.034216+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:50:01.034216+0000","last_peered":"2024-09-13T14:50:01.034216+0000","last_clean":"2024-09-13T14:50:01.034216+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:50:01.034216+0000","last_undegraded":"2024-09-13T14:50:01.034216+0000","last_fullsized":"2024-09-13T14:50:01.034216+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":583,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.526823+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:50:09.526823+0000","last_peered":"2024-09-13T14:50:09.526823+0000","last_clean":"2024-09-13T14:50:09.526823+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:50:09.526823+0000","last_undegraded":"2024-09-13T14:50:09.526823+0000","last_fullsized":"2024-09-13T14:50:09.526823+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":378,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.509473+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:50:00.509473+0000","last_peered":"2024-09-13T14:50:00.509473+0000","last_clean":"2024-09-13T14:50:00.509473+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:50:00.509473+0000","last_undegraded":"2024-09-13T14:50:00.509473+0000","last_fullsized":"2024-09-13T14:50:00.509473+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":344,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508881+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:50:00.508881+0000","last_peered":"2024-09-13T14:50:00.508881+0000","last_clean":"2024-09-13T14:50:00.508881+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:50:00.508881+0000","last_undegraded":"2024-09-13T14:50:00.508881+0000","last_fullsized":"2024-09-13T14:50:00.508881+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":588,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420295+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:50:10.420295+0000","last_peered":"2024-09-13T14:50:10.420295+0000","last_clean":"2024-09-13T14:50:10.420295+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:50:10.420295+0000","last_undegraded":"2024-09-13T14:50:10.420295+0000","last_fullsized":"2024-09-13T14:50:10.420295+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":639,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495020+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:50:10.495020+0000","last_peered":"2024-09-13T14:50:10.495020+0000","last_clean":"2024-09-13T14:50:10.495020+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:50:10.495020+0000","last_undegraded":"2024-09-13T14:50:10.495020+0000","last_fullsized":"2024-09-13T14:50:10.495020+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":644,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420174+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:50:10.420174+0000","last_peered":"2024-09-13T14:50:10.420174+0000","last_clean":"2024-09-13T14:50:10.420174+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:50:10.420174+0000","last_undegraded":"2024-09-13T14:50:10.420174+0000","last_fullsized":"2024-09-13T14:50:10.420174+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":376,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033464+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:50:01.033464+0000","last_peered":"2024-09-13T14:50:01.033464+0000","last_clean":"2024-09-13T14:50:01.033464+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:50:01.033464+0000","last_undegraded":"2024-09-13T14:50:01.033464+0000","last_fullsized":"2024-09-13T14:50:01.033464+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":661,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420302+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:50:10.420302+0000","last_peered":"2024-09-13T14:50:10.420302+0000","last_clean":"2024-09-13T14:50:10.420302+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:50:10.420302+0000","last_undegraded":"2024-09-13T14:50:10.420302+0000","last_fullsized":"2024-09-13T14:50:10.420302+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":632,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495447+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:50:10.495447+0000","last_peered":"2024-09-13T14:50:10.495447+0000","last_clean":"2024-09-13T14:50:10.495447+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:50:10.495447+0000","last_undegraded":"2024-09-13T14:50:10.495447+0000","last_fullsized":"2024-09-13T14:50:10.495447+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":608,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.526566+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:50:09.526566+0000","last_peered":"2024-09-13T14:50:09.526566+0000","last_clean":"2024-09-13T14:50:09.526566+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:50:09.526566+0000","last_undegraded":"2024-09-13T14:50:09.526566+0000","last_fullsized":"2024-09-13T14:50:09.526566+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":657,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420054+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:50:10.420054+0000","last_peered":"2024-09-13T14:50:10.420054+0000","last_clean":"2024-09-13T14:50:10.420054+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:50:10.420054+0000","last_undegraded":"2024-09-13T14:50:10.420054+0000","last_fullsized":"2024-09-13T14:50:10.420054+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":77,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033931+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:50:01.033931+0000","last_peered":"2024-09-13T14:50:01.033931+0000","last_clean":"2024-09-13T14:50:01.033931+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:50:01.033931+0000","last_undegraded":"2024-09-13T14:50:01.033931+0000","last_fullsized":"2024-09-13T14:50:01.033931+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":554,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.527230+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:50:09.527230+0000","last_peered":"2024-09-13T14:50:09.527230+0000","last_clean":"2024-09-13T14:50:09.527230+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:50:09.527230+0000","last_undegraded":"2024-09-13T14:50:09.527230+0000","last_fullsized":"2024-09-13T14:50:09.527230+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":635,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494714+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:50:10.494714+0000","last_peered":"2024-09-13T14:50:10.494714+0000","last_clean":"2024-09-13T14:50:10.494714+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:50:10.494714+0000","last_undegraded":"2024-09-13T14:50:10.494714+0000","last_fullsized":"2024-09-13T14:50:10.494714+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033815+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:50:01.033815+0000","last_peered":"2024-09-13T14:50:01.033815+0000","last_clean":"2024-09-13T14:50:01.033815+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:50:01.033815+0000","last_undegraded":"2024-09-13T14:50:01.033815+0000","last_fullsized":"2024-09-13T14:50:01.033815+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":344,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508402+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:50:00.508402+0000","last_peered":"2024-09-13T14:50:00.508402+0000","last_clean":"2024-09-13T14:50:00.508402+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:50:00.508402+0000","last_undegraded":"2024-09-13T14:50:00.508402+0000","last_fullsized":"2024-09-13T14:50:00.508402+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033847+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:50:01.033847+0000","last_peered":"2024-09-13T14:50:01.033847+0000","last_clean":"2024-09-13T14:50:01.033847+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:50:01.033847+0000","last_undegraded":"2024-09-13T14:50:01.033847+0000","last_fullsized":"2024-09-13T14:50:01.033847+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":310,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:00.508858+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:50:00.508858+0000","last_peered":"2024-09-13T14:50:00.508858+0000","last_clean":"2024-09-13T14:50:00.508858+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:50:00.508858+0000","last_undegraded":"2024-09-13T14:50:00.508858+0000","last_fullsized":"2024-09-13T14:50:00.508858+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":653,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420650+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:50:10.420650+0000","last_peered":"2024-09-13T14:50:10.420650+0000","last_clean":"2024-09-13T14:50:10.420650+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:50:10.420650+0000","last_undegraded":"2024-09-13T14:50:10.420650+0000","last_fullsized":"2024-09-13T14:50:10.420650+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":641,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.420915+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:50:10.420915+0000","last_peered":"2024-09-13T14:50:10.420915+0000","last_clean":"2024-09-13T14:50:10.420915+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:50:10.420915+0000","last_undegraded":"2024-09-13T14:50:10.420915+0000","last_fullsized":"2024-09-13T14:50:10.420915+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-09-13T14:50:01.033172+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:50:01.033172+0000","last_peered":"2024-09-13T14:50:01.033172+0000","last_clean":"2024-09-13T14:50:01.033172+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:50:01.033172+0000","last_undegraded":"2024-09-13T14:50:01.033172+0000","last_fullsized":"2024-09-13T14:50:01.033172+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":645,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495016+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:50:10.495016+0000","last_peered":"2024-09-13T14:50:10.495016+0000","last_clean":"2024-09-13T14:50:10.495016+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:50:10.495016+0000","last_undegraded":"2024-09-13T14:50:10.495016+0000","last_fullsized":"2024-09-13T14:50:10.495016+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":137494528,"num_objects":2098,"num_object_clones":0,"num_object_copies":6294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2098,"num_write_kb":134272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":417988608,"data_stored":417988608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2098,"ondisk_log_size":2098,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397969,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896984,"kb_used_data":9292,"kb_used_omap":1,"kb_used_meta":887678,"kb_avail":92848168,"statfs":{"total":95995035648,"available":95076524032,"internally_reserved":0,"allocated":9515008,"data_stored":7472129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":908982616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594197,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1218692,"kb_used_data":311740,"kb_used_omap":1,"kb_used_meta":906942,"kb_avail":92526460,"statfs":{"total":95995035648,"available":94747095040,"internally_reserved":0,"allocated":319221760,"data_stored":317194563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":928708952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665651,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333828,"kb_used_data":5432,"kb_used_omap":9,"kb_used_meta":328374,"kb_avail":93411324,"statfs":{"total":95995035648,"available":95653195776,"internally_reserved":0,"allocated":5562368,"data_stored":3535574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":336255176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986653,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900564,"kb_used_data":4872,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92844588,"statfs":{"total":95995035648,"available":95072858112,"internally_reserved":0,"allocated":4988928,"data_stored":2950075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697212,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1096116,"kb_used_data":319916,"kb_used_omap":4,"kb_used_meta":776187,"kb_avail":92649036,"statfs":{"total":95995035648,"available":94872612864,"internally_reserved":0,"allocated":327593984,"data_stored":325570885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":794815563},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860739,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":644296,"kb_used_data":5116,"kb_used_omap":8,"kb_used_meta":639159,"kb_avail":93100856,"statfs":{"total":95995035648,"available":95335276544,"internally_reserved":0,"allocated":5238784,"data_stored":3208489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":654499660},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020411,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597948,"kb_used_data":262452,"kb_used_omap":9,"kb_used_meta":335478,"kb_avail":93147204,"statfs":{"total":95995035648,"available":95382736896,"internally_reserved":0,"allocated":268750848,"data_stored":266727608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":343530309},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187801,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":705404,"kb_used_data":73332,"kb_used_omap":7,"kb_used_meta":632056,"kb_avail":93039748,"statfs":{"total":95995035648,"available":95272701952,"internally_reserved":0,"allocated":75091968,"data_stored":73068994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":647225675},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":80805888,"data_stored":80805888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":311099392,"data_stored":311099392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":26083328,"data_stored":26083328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:50:24.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:24.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:24 smithi017 ceph-mon[103001]: pgmap v1266: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-13T14:50:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:24 smithi017 ceph-mon[107556]: pgmap v1266: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-13T14:50:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:24 smithi189 ceph-mon[90992]: pgmap v1266: 99 pgs: 99 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-13T14:50:25.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:25.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:25.340 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:25.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:25.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:25.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:25.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:25.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:25.799 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 33039 33023 158.736 118.188 0.00707755 0.00629409 2024-09-13T14:50:25.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:26.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.263 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:26.264 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:26.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:26.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:26.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:26.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:26.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.790 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:26.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 34903 34887 155.705 116.5 0.00708305 0.00641653 2024-09-13T14:50:26.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:26.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:26.929 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:26 smithi017 ceph-mon[103001]: pgmap v1267: 99 pgs: 99 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:50:26.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:26 smithi017 ceph-mon[107556]: pgmap v1267: 99 pgs: 99 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:50:26.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:26 smithi189 ceph-mon[90992]: pgmap v1267: 99 pgs: 99 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-13T14:50:27.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:27.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:27.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:27.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:27.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:27.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:27.538 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.569 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:27.593 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:27] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-09-13T14:50:27.670 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:27.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:27.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 36796 36780 153.212 118.312 0.00577151 0.00652194 2024-09-13T14:50:27.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:27.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:27.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:50:27.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:27.847 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:27.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:27.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:28.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:28.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:28.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:28.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:28.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.387 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:28.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:28.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:28.521 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:28.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:28.656 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:28.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:28.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:28.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:28.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 38703 38687 151.085 119.188 0.0126741 0.00661284 2024-09-13T14:50:28.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:28 smithi017 ceph-mon[103001]: pgmap v1268: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:50:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:28 smithi017 ceph-mon[107556]: pgmap v1268: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:50:29.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:28 smithi189 ceph-mon[90992]: pgmap v1268: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-13T14:50:29.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:29.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:29.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:50:29.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:29.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:29.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:29.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:29.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:29.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.470 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:29.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:29.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:29.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:29.744 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:50:29.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:29.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:29.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 40361 40345 148.291 103.625 0.00845285 0.00673894 2024-09-13T14:50:29.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3881317970' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:29.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:29.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3881317970' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:29.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:29.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:29.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:30.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3881317970' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:30.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:30.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:30.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:30.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:30.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:30.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:30.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:30.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:30.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:30.701 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:30.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 42111 42095 146.129 109.375 0.00808513 0.00683836 2024-09-13T14:50:30.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:30.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:30.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:30 smithi189 ceph-mon[90992]: pgmap v1269: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-09-13T14:50:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:30 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:30 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:30 smithi189 ceph-mon[90992]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:50:31.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:31.048 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[103001]: pgmap v1269: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[103001]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[107556]: pgmap v1269: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:31.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:30 smithi017 ceph-mon[107556]: osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:50:31.050 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:50:31.050 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:31.053 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:31.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:50:31.097 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:31.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.130 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:50:31.131 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:31.173 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:50:31.173 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:31.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:50:31.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:31.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:50:31.241 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:31.271 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:50:31.271 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:31.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:50:31.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:31.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:31.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:31.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:31.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 43900 43884 144.323 111.812 0.00330865 0.00692359 2024-09-13T14:50:31.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.902 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1932321293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:31.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1932321293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:31.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1932321293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:32.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:32.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:32.162 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:32.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:32.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:32.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:32.429 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:32.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:32.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:32.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:32.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:32.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:32.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:50:32.801634+0000 min lat: 0.00120371 max lat: 0.172172 avg lat: 0.00701346 2024-09-13T14:50:32.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:50:32.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 45622 45606 142.488 107.625 0.00760142 0.00701346 2024-09-13T14:50:32.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:32.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:32.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:32 smithi189 ceph-mon[90992]: pgmap v1271: 99 pgs: 99 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T14:50:33.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:32 smithi017 ceph-mon[103001]: pgmap v1271: 99 pgs: 99 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T14:50:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:32 smithi017 ceph-mon[107556]: pgmap v1271: 99 pgs: 99 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-13T14:50:33.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:33.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:33.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:33.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.540 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:33.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:50:33.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.801 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 47473 47457 141.211 115.688 0.0066873 0.00707531 2024-09-13T14:50:33.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:33.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.957 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:33.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:34.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:34.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:34.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:34.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:34.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:34.400 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:34.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:34.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:34.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:34.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:34.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:34.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:34.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 49130 49114 139.499 103.562 0.00871881 0.00716207 2024-09-13T14:50:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:34.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:34.898 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:50:34.899 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:50:34.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:34 smithi189 ceph-mon[90992]: pgmap v1272: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-13T14:50:35.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:34 smithi017 ceph-mon[103001]: pgmap v1272: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-13T14:50:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:34 smithi017 ceph-mon[107556]: pgmap v1272: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-13T14:50:35.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:35.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:35.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:35.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:35.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 50656 50640 137.581 95.375 0.0109007 0.00726374 2024-09-13T14:50:35.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:35.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:35.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:35.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:35.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:36.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:36.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:36.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:36.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 52305 52289 136.142 103.062 0.011677 0.00734004 2024-09-13T14:50:36.822 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:36.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:36.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:37.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:37.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:36 smithi017 ceph-mon[103001]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:36 smithi017 ceph-mon[103001]: pgmap v1273: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:50:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:36 smithi017 ceph-mon[107556]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:36 smithi017 ceph-mon[107556]: pgmap v1273: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:50:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:37.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:36 smithi189 ceph-mon[90992]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:37.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:36 smithi189 ceph-mon[90992]: pgmap v1273: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:50:37.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:37.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:37.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:37.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:37.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:37.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:37] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-09-13T14:50:37.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:37.769 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:37.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:37.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 53786 53770 134.399 92.5625 0.0222147 0.00743092 2024-09-13T14:50:37.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:38.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:38.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:38.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:38.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:38.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:38.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:38.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:38.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 55465 55449 133.265 104.938 0.0154372 0.00749856 2024-09-13T14:50:38.821 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.857 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:38.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:38.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:38.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:39.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:38 smithi017 ceph-mon[103001]: pgmap v1274: 99 pgs: 99 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-13T14:50:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:38 smithi017 ceph-mon[107556]: pgmap v1274: 99 pgs: 99 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-13T14:50:39.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:38 smithi189 ceph-mon[90992]: pgmap v1274: 99 pgs: 99 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-13T14:50:39.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:39.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:39.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:39.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:39.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:39.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:39.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.801 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:39.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:39.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 56897 56881 131.644 89.5 0.00751117 0.00759092 2024-09-13T14:50:39.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.936 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:39 smithi017 ceph-mon[103001]: pgmap v1275: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:50:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:39 smithi017 ceph-mon[107556]: pgmap v1275: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:50:40.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:40.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:40.178 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:40.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:40.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:39 smithi189 ceph-mon[90992]: pgmap v1275: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:50:40.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:40.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:40.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.606 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:40.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:40.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 58387 58371 130.268 93.125 0.00893746 0.00767114 2024-09-13T14:50:40.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:40.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:40.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.016 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:41.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:41.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:41.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:41.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:41.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:41.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:41.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 59805 59789 128.832 88.625 0.0155813 0.00775689 2024-09-13T14:50:41.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:41.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:41.927 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1272,"stamp":"2024-09-13T14:50:33.318752+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2826901453,"num_objects":43090,"num_object_clones":0,"num_object_copies":129270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43090,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":43969,"num_write_kb":2768591,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45194,"ondisk_log_size":45194,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14410308,"kb_used_data":8821240,"kb_used_omap":44,"kb_used_meta":5588947,"kb_avail":735550908,"statfs":{"total":767960285184,"available":753204129792,"internally_reserved":0,"allocated":9032949760,"data_stored":9016683137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5723081975},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1220542464,"num_objects":18624,"num_object_clones":0,"num_object_copies":55872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18624,"num_write_kb":1191936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003158"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":646,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737614+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:50:29.737614+0000","last_peered":"2024-09-13T14:50:29.737614+0000","last_clean":"2024-09-13T14:50:29.737614+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:50:29.737614+0000","last_undegraded":"2024-09-13T14:50:29.737614+0000","last_fullsized":"2024-09-13T14:50:29.737614+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":378,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495076+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:50:10.495076+0000","last_peered":"2024-09-13T14:50:10.495076+0000","last_clean":"2024-09-13T14:50:10.495076+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:50:10.495076+0000","last_undegraded":"2024-09-13T14:50:10.495076+0000","last_fullsized":"2024-09-13T14:50:10.495076+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529983+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:50:09.529983+0000","last_peered":"2024-09-13T14:50:09.529983+0000","last_clean":"2024-09-13T14:50:09.529983+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:50:09.529983+0000","last_undegraded":"2024-09-13T14:50:09.529983+0000","last_fullsized":"2024-09-13T14:50:09.529983+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":629,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495070+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:50:10.495070+0000","last_peered":"2024-09-13T14:50:10.495070+0000","last_clean":"2024-09-13T14:50:10.495070+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:50:10.495070+0000","last_undegraded":"2024-09-13T14:50:10.495070+0000","last_fullsized":"2024-09-13T14:50:10.495070+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":629,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495360+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:50:10.495360+0000","last_peered":"2024-09-13T14:50:10.495360+0000","last_clean":"2024-09-13T14:50:10.495360+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:50:10.495360+0000","last_undegraded":"2024-09-13T14:50:10.495360+0000","last_fullsized":"2024-09-13T14:50:10.495360+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494628+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:50:10.494628+0000","last_peered":"2024-09-13T14:50:10.494628+0000","last_clean":"2024-09-13T14:50:10.494628+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:50:10.494628+0000","last_undegraded":"2024-09-13T14:50:10.494628+0000","last_fullsized":"2024-09-13T14:50:10.494628+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":645,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.741655+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:50:29.741655+0000","last_peered":"2024-09-13T14:50:29.741655+0000","last_clean":"2024-09-13T14:50:29.741655+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:50:29.741655+0000","last_undegraded":"2024-09-13T14:50:29.741655+0000","last_fullsized":"2024-09-13T14:50:29.741655+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":369,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529522+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:50:09.529522+0000","last_peered":"2024-09-13T14:50:09.529522+0000","last_clean":"2024-09-13T14:50:09.529522+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:50:09.529522+0000","last_undegraded":"2024-09-13T14:50:09.529522+0000","last_fullsized":"2024-09-13T14:50:09.529522+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"379'2492","reported_seq":2509,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.137228+0000","last_change":"2024-09-13T14:49:59.509935+0000","last_active":"2024-09-13T14:50:29.137228+0000","last_peered":"2024-09-13T14:50:29.137228+0000","last_clean":"2024-09-13T14:50:29.137228+0000","last_became_active":"2024-09-13T14:49:59.509619+0000","last_became_peered":"2024-09-13T14:49:59.509619+0000","last_unstale":"2024-09-13T14:50:29.137228+0000","last_undegraded":"2024-09-13T14:50:29.137228+0000","last_fullsized":"2024-09-13T14:50:29.137228+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2492,"log_dups_size":0,"ondisk_log_size":2492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:39:57.083056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2492,"num_write_kb":159488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":373,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.496206+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:50:10.496206+0000","last_peered":"2024-09-13T14:50:10.496206+0000","last_clean":"2024-09-13T14:50:10.496206+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:50:10.496206+0000","last_undegraded":"2024-09-13T14:50:10.496206+0000","last_fullsized":"2024-09-13T14:50:10.496206+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":292,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529198+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:50:09.529198+0000","last_peered":"2024-09-13T14:50:09.529198+0000","last_clean":"2024-09-13T14:50:09.529198+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:50:09.529198+0000","last_undegraded":"2024-09-13T14:50:09.529198+0000","last_fullsized":"2024-09-13T14:50:09.529198+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"380'2598","reported_seq":2617,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.156087+0000","last_change":"2024-09-13T14:49:59.755413+0000","last_active":"2024-09-13T14:50:30.156087+0000","last_peered":"2024-09-13T14:50:30.156087+0000","last_clean":"2024-09-13T14:50:30.156087+0000","last_became_active":"2024-09-13T14:49:59.755054+0000","last_became_peered":"2024-09-13T14:49:59.755054+0000","last_unstale":"2024-09-13T14:50:30.156087+0000","last_undegraded":"2024-09-13T14:50:30.156087+0000","last_fullsized":"2024-09-13T14:50:30.156087+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2598,"log_dups_size":0,"ondisk_log_size":2598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:24.559415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170262528,"num_objects":2598,"num_object_clones":0,"num_object_copies":7794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2598,"num_write_kb":166272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":391,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495228+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:50:10.495228+0000","last_peered":"2024-09-13T14:50:10.495228+0000","last_clean":"2024-09-13T14:50:10.495228+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:50:10.495228+0000","last_undegraded":"2024-09-13T14:50:10.495228+0000","last_fullsized":"2024-09-13T14:50:10.495228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494956+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:50:10.494956+0000","last_peered":"2024-09-13T14:50:10.494956+0000","last_clean":"2024-09-13T14:50:10.494956+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:50:10.494956+0000","last_undegraded":"2024-09-13T14:50:10.494956+0000","last_fullsized":"2024-09-13T14:50:10.494956+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"380'2764","reported_seq":2776,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.986593+0000","last_change":"2024-09-13T14:49:59.509888+0000","last_active":"2024-09-13T14:50:32.986593+0000","last_peered":"2024-09-13T14:50:32.986593+0000","last_clean":"2024-09-13T14:50:32.986593+0000","last_became_active":"2024-09-13T14:49:59.509562+0000","last_became_peered":"2024-09-13T14:49:59.509562+0000","last_unstale":"2024-09-13T14:50:32.986593+0000","last_undegraded":"2024-09-13T14:50:32.986593+0000","last_fullsized":"2024-09-13T14:50:32.986593+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2764,"log_dups_size":0,"ondisk_log_size":2764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:53.170631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181141504,"num_objects":2764,"num_object_clones":0,"num_object_copies":8292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2764,"num_write_kb":176896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495843+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:50:10.495843+0000","last_peered":"2024-09-13T14:50:10.495843+0000","last_clean":"2024-09-13T14:50:10.495843+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:50:10.495843+0000","last_undegraded":"2024-09-13T14:50:10.495843+0000","last_fullsized":"2024-09-13T14:50:10.495843+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":360,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529729+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:50:09.529729+0000","last_peered":"2024-09-13T14:50:09.529729+0000","last_clean":"2024-09-13T14:50:09.529729+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:50:09.529729+0000","last_undegraded":"2024-09-13T14:50:09.529729+0000","last_fullsized":"2024-09-13T14:50:09.529729+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"380'2896","reported_seq":2908,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.986665+0000","last_change":"2024-09-13T14:50:00.002234+0000","last_active":"2024-09-13T14:50:32.986665+0000","last_peered":"2024-09-13T14:50:32.986665+0000","last_clean":"2024-09-13T14:50:32.986665+0000","last_became_active":"2024-09-13T14:50:00.001900+0000","last_became_peered":"2024-09-13T14:50:00.001900+0000","last_unstale":"2024-09-13T14:50:32.986665+0000","last_undegraded":"2024-09-13T14:50:32.986665+0000","last_fullsized":"2024-09-13T14:50:32.986665+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2896,"log_dups_size":0,"ondisk_log_size":2896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:03.241074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189792256,"num_objects":2896,"num_object_clones":0,"num_object_copies":8688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2896,"num_write_kb":185344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529486+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:50:09.529486+0000","last_peered":"2024-09-13T14:50:09.529486+0000","last_clean":"2024-09-13T14:50:09.529486+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:50:09.529486+0000","last_undegraded":"2024-09-13T14:50:09.529486+0000","last_fullsized":"2024-09-13T14:50:09.529486+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":655,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494456+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:50:10.494456+0000","last_peered":"2024-09-13T14:50:10.494456+0000","last_clean":"2024-09-13T14:50:10.494456+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:50:10.494456+0000","last_undegraded":"2024-09-13T14:50:10.494456+0000","last_fullsized":"2024-09-13T14:50:10.494456+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"380'2654","reported_seq":2673,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.879161+0000","last_change":"2024-09-13T14:49:59.511081+0000","last_active":"2024-09-13T14:50:30.879161+0000","last_peered":"2024-09-13T14:50:30.879161+0000","last_clean":"2024-09-13T14:50:30.879161+0000","last_became_active":"2024-09-13T14:49:59.510738+0000","last_became_peered":"2024-09-13T14:49:59.510738+0000","last_unstale":"2024-09-13T14:50:30.879161+0000","last_undegraded":"2024-09-13T14:50:30.879161+0000","last_fullsized":"2024-09-13T14:50:30.879161+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2654,"log_dups_size":0,"ondisk_log_size":2654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:53.029264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173932544,"num_objects":2654,"num_object_clones":0,"num_object_copies":7962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2654,"num_write_kb":169856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":491,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494923+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:50:10.494923+0000","last_peered":"2024-09-13T14:50:10.494923+0000","last_clean":"2024-09-13T14:50:10.494923+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:50:10.494923+0000","last_undegraded":"2024-09-13T14:50:10.494923+0000","last_fullsized":"2024-09-13T14:50:10.494923+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":655,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737115+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:50:29.737115+0000","last_peered":"2024-09-13T14:50:29.737115+0000","last_clean":"2024-09-13T14:50:29.737115+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:50:29.737115+0000","last_undegraded":"2024-09-13T14:50:29.737115+0000","last_fullsized":"2024-09-13T14:50:29.737115+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494715+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:50:10.494715+0000","last_peered":"2024-09-13T14:50:10.494715+0000","last_clean":"2024-09-13T14:50:10.494715+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:50:10.494715+0000","last_undegraded":"2024-09-13T14:50:10.494715+0000","last_fullsized":"2024-09-13T14:50:10.494715+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"380'2543","reported_seq":2562,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.165659+0000","last_change":"2024-09-13T14:50:00.002352+0000","last_active":"2024-09-13T14:50:30.165659+0000","last_peered":"2024-09-13T14:50:30.165659+0000","last_clean":"2024-09-13T14:50:30.165659+0000","last_became_active":"2024-09-13T14:50:00.002036+0000","last_became_peered":"2024-09-13T14:50:00.002036+0000","last_unstale":"2024-09-13T14:50:30.165659+0000","last_undegraded":"2024-09-13T14:50:30.165659+0000","last_fullsized":"2024-09-13T14:50:30.165659+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2543,"log_dups_size":0,"ondisk_log_size":2543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:29:00.760558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166658048,"num_objects":2543,"num_object_clones":0,"num_object_copies":7629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2543,"num_write_kb":162752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":205,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529314+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:50:09.529314+0000","last_peered":"2024-09-13T14:50:09.529314+0000","last_clean":"2024-09-13T14:50:09.529314+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:50:09.529314+0000","last_undegraded":"2024-09-13T14:50:09.529314+0000","last_fullsized":"2024-09-13T14:50:09.529314+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":381,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529592+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:50:09.529592+0000","last_peered":"2024-09-13T14:50:09.529592+0000","last_clean":"2024-09-13T14:50:09.529592+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:50:09.529592+0000","last_undegraded":"2024-09-13T14:50:09.529592+0000","last_fullsized":"2024-09-13T14:50:09.529592+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":93,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.496240+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:50:10.496240+0000","last_peered":"2024-09-13T14:50:10.496240+0000","last_clean":"2024-09-13T14:50:10.496240+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:50:10.496240+0000","last_undegraded":"2024-09-13T14:50:10.496240+0000","last_fullsized":"2024-09-13T14:50:10.496240+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"380'2900","reported_seq":2911,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.670832+0000","last_change":"2024-09-13T14:49:59.511648+0000","last_active":"2024-09-13T14:50:32.670832+0000","last_peered":"2024-09-13T14:50:32.670832+0000","last_clean":"2024-09-13T14:50:32.670832+0000","last_became_active":"2024-09-13T14:49:59.511315+0000","last_became_peered":"2024-09-13T14:49:59.511315+0000","last_unstale":"2024-09-13T14:50:32.670832+0000","last_undegraded":"2024-09-13T14:50:32.670832+0000","last_fullsized":"2024-09-13T14:50:32.670832+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2900,"log_dups_size":0,"ondisk_log_size":2900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:43.509125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190054400,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2900,"num_write_kb":185600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495605+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:50:10.495605+0000","last_peered":"2024-09-13T14:50:10.495605+0000","last_clean":"2024-09-13T14:50:10.495605+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:50:10.495605+0000","last_undegraded":"2024-09-13T14:50:10.495605+0000","last_fullsized":"2024-09-13T14:50:10.495605+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495247+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:50:10.495247+0000","last_peered":"2024-09-13T14:50:10.495247+0000","last_clean":"2024-09-13T14:50:10.495247+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:50:10.495247+0000","last_undegraded":"2024-09-13T14:50:10.495247+0000","last_fullsized":"2024-09-13T14:50:10.495247+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"380'2695","reported_seq":2714,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.847262+0000","last_change":"2024-09-13T14:50:00.002592+0000","last_active":"2024-09-13T14:50:30.847262+0000","last_peered":"2024-09-13T14:50:30.847262+0000","last_clean":"2024-09-13T14:50:30.847262+0000","last_became_active":"2024-09-13T14:50:00.002051+0000","last_became_peered":"2024-09-13T14:50:00.002051+0000","last_unstale":"2024-09-13T14:50:30.847262+0000","last_undegraded":"2024-09-13T14:50:30.847262+0000","last_fullsized":"2024-09-13T14:50:30.847262+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2695,"log_dups_size":0,"ondisk_log_size":2695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:28:44.776933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176619520,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2695,"num_write_kb":172480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":198,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530009+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:50:09.530009+0000","last_peered":"2024-09-13T14:50:09.530009+0000","last_clean":"2024-09-13T14:50:09.530009+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:50:09.530009+0000","last_undegraded":"2024-09-13T14:50:09.530009+0000","last_fullsized":"2024-09-13T14:50:09.530009+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":648,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.738304+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:50:29.738304+0000","last_peered":"2024-09-13T14:50:29.738304+0000","last_clean":"2024-09-13T14:50:29.738304+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:50:29.738304+0000","last_undegraded":"2024-09-13T14:50:29.738304+0000","last_fullsized":"2024-09-13T14:50:29.738304+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":610,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745311+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:50:29.745311+0000","last_peered":"2024-09-13T14:50:29.745311+0000","last_clean":"2024-09-13T14:50:29.745311+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:50:29.745311+0000","last_undegraded":"2024-09-13T14:50:29.745311+0000","last_fullsized":"2024-09-13T14:50:29.745311+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"380'2587","reported_seq":2606,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.878691+0000","last_change":"2024-09-13T14:49:59.509238+0000","last_active":"2024-09-13T14:50:30.878691+0000","last_peered":"2024-09-13T14:50:30.878691+0000","last_clean":"2024-09-13T14:50:30.878691+0000","last_became_active":"2024-09-13T14:49:59.508811+0000","last_became_peered":"2024-09-13T14:49:59.508811+0000","last_unstale":"2024-09-13T14:50:30.878691+0000","last_undegraded":"2024-09-13T14:50:30.878691+0000","last_fullsized":"2024-09-13T14:50:30.878691+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2587,"log_dups_size":0,"ondisk_log_size":2587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:43.422288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2587,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":482,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495127+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:50:10.495127+0000","last_peered":"2024-09-13T14:50:10.495127+0000","last_clean":"2024-09-13T14:50:10.495127+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:50:10.495127+0000","last_undegraded":"2024-09-13T14:50:10.495127+0000","last_fullsized":"2024-09-13T14:50:10.495127+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":904,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529474+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:50:09.529474+0000","last_peered":"2024-09-13T14:50:09.529474+0000","last_clean":"2024-09-13T14:50:09.529474+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:50:09.529474+0000","last_undegraded":"2024-09-13T14:50:09.529474+0000","last_fullsized":"2024-09-13T14:50:09.529474+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":601,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.738156+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:50:29.738156+0000","last_peered":"2024-09-13T14:50:29.738156+0000","last_clean":"2024-09-13T14:50:29.738156+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:50:29.738156+0000","last_undegraded":"2024-09-13T14:50:29.738156+0000","last_fullsized":"2024-09-13T14:50:29.738156+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530400+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:50:09.530400+0000","last_peered":"2024-09-13T14:50:09.530400+0000","last_clean":"2024-09-13T14:50:09.530400+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:50:09.530400+0000","last_undegraded":"2024-09-13T14:50:09.530400+0000","last_fullsized":"2024-09-13T14:50:09.530400+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"379'2436","reported_seq":2453,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.123761+0000","last_change":"2024-09-13T14:49:59.507963+0000","last_active":"2024-09-13T14:50:29.123761+0000","last_peered":"2024-09-13T14:50:29.123761+0000","last_clean":"2024-09-13T14:50:29.123761+0000","last_became_active":"2024-09-13T14:49:59.507611+0000","last_became_peered":"2024-09-13T14:49:59.507611+0000","last_unstale":"2024-09-13T14:50:29.123761+0000","last_undegraded":"2024-09-13T14:50:29.123761+0000","last_fullsized":"2024-09-13T14:50:29.123761+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2436,"log_dups_size":0,"ondisk_log_size":2436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:32.773088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159645696,"num_objects":2436,"num_object_clones":0,"num_object_copies":7308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2436,"num_write_kb":155904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":455,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744800+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:50:29.744800+0000","last_peered":"2024-09-13T14:50:29.744800+0000","last_clean":"2024-09-13T14:50:29.744800+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:50:29.744800+0000","last_undegraded":"2024-09-13T14:50:29.744800+0000","last_fullsized":"2024-09-13T14:50:29.744800+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":770,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495035+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:50:10.495035+0000","last_peered":"2024-09-13T14:50:10.495035+0000","last_clean":"2024-09-13T14:50:10.495035+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:50:10.495035+0000","last_undegraded":"2024-09-13T14:50:10.495035+0000","last_fullsized":"2024-09-13T14:50:10.495035+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":648,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495447+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:50:10.495447+0000","last_peered":"2024-09-13T14:50:10.495447+0000","last_clean":"2024-09-13T14:50:10.495447+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:50:10.495447+0000","last_undegraded":"2024-09-13T14:50:10.495447+0000","last_fullsized":"2024-09-13T14:50:10.495447+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":653,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495586+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:50:10.495586+0000","last_peered":"2024-09-13T14:50:10.495586+0000","last_clean":"2024-09-13T14:50:10.495586+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:50:10.495586+0000","last_undegraded":"2024-09-13T14:50:10.495586+0000","last_fullsized":"2024-09-13T14:50:10.495586+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"380'2810","reported_seq":2822,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.970144+0000","last_change":"2024-09-13T14:49:59.509683+0000","last_active":"2024-09-13T14:50:32.970144+0000","last_peered":"2024-09-13T14:50:32.970144+0000","last_clean":"2024-09-13T14:50:32.970144+0000","last_became_active":"2024-09-13T14:49:59.509390+0000","last_became_peered":"2024-09-13T14:49:59.509390+0000","last_unstale":"2024-09-13T14:50:32.970144+0000","last_undegraded":"2024-09-13T14:50:32.970144+0000","last_fullsized":"2024-09-13T14:50:32.970144+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2810,"log_dups_size":0,"ondisk_log_size":2810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:08.503709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184156160,"num_objects":2810,"num_object_clones":0,"num_object_copies":8430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2810,"num_write_kb":179840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":511,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495279+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:50:10.495279+0000","last_peered":"2024-09-13T14:50:10.495279+0000","last_clean":"2024-09-13T14:50:10.495279+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:50:10.495279+0000","last_undegraded":"2024-09-13T14:50:10.495279+0000","last_fullsized":"2024-09-13T14:50:10.495279+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":619,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737210+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:50:29.737210+0000","last_peered":"2024-09-13T14:50:29.737210+0000","last_clean":"2024-09-13T14:50:29.737210+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:50:29.737210+0000","last_undegraded":"2024-09-13T14:50:29.737210+0000","last_fullsized":"2024-09-13T14:50:29.737210+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":587,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744590+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:50:29.744590+0000","last_peered":"2024-09-13T14:50:29.744590+0000","last_clean":"2024-09-13T14:50:29.744590+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:50:29.744590+0000","last_undegraded":"2024-09-13T14:50:29.744590+0000","last_fullsized":"2024-09-13T14:50:29.744590+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530782+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:50:09.530782+0000","last_peered":"2024-09-13T14:50:09.530782+0000","last_clean":"2024-09-13T14:50:09.530782+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:50:09.530782+0000","last_undegraded":"2024-09-13T14:50:09.530782+0000","last_fullsized":"2024-09-13T14:50:09.530782+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"379'2423","reported_seq":2440,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.120612+0000","last_change":"2024-09-13T14:49:59.510827+0000","last_active":"2024-09-13T14:50:29.120612+0000","last_peered":"2024-09-13T14:50:29.120612+0000","last_clean":"2024-09-13T14:50:29.120612+0000","last_became_active":"2024-09-13T14:49:59.509711+0000","last_became_peered":"2024-09-13T14:49:59.509711+0000","last_unstale":"2024-09-13T14:50:29.120612+0000","last_undegraded":"2024-09-13T14:50:29.120612+0000","last_fullsized":"2024-09-13T14:50:29.120612+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2423,"log_dups_size":0,"ondisk_log_size":2423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:17.400887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2423,"num_write_kb":155072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":198,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529690+0000","last_change":"2024-09-13T14:49:47.727307+0000","last_active":"2024-09-13T14:50:09.529690+0000","last_peered":"2024-09-13T14:50:09.529690+0000","last_clean":"2024-09-13T14:50:09.529690+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:50:09.529690+0000","last_undegraded":"2024-09-13T14:50:09.529690+0000","last_fullsized":"2024-09-13T14:50:09.529690+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:30:33.609834+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":995,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530057+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:50:09.530057+0000","last_peered":"2024-09-13T14:50:09.530057+0000","last_clean":"2024-09-13T14:50:09.530057+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:50:09.530057+0000","last_undegraded":"2024-09-13T14:50:09.530057+0000","last_fullsized":"2024-09-13T14:50:09.530057+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494915+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:50:10.494915+0000","last_peered":"2024-09-13T14:50:10.494915+0000","last_clean":"2024-09-13T14:50:10.494915+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:50:10.494915+0000","last_undegraded":"2024-09-13T14:50:10.494915+0000","last_fullsized":"2024-09-13T14:50:10.494915+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":608,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744137+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:50:29.744137+0000","last_peered":"2024-09-13T14:50:29.744137+0000","last_clean":"2024-09-13T14:50:29.744137+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:50:29.744137+0000","last_undegraded":"2024-09-13T14:50:29.744137+0000","last_fullsized":"2024-09-13T14:50:29.744137+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"380'2874","reported_seq":2886,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.980704+0000","last_change":"2024-09-13T14:49:59.507705+0000","last_active":"2024-09-13T14:50:32.980704+0000","last_peered":"2024-09-13T14:50:32.980704+0000","last_clean":"2024-09-13T14:50:32.980704+0000","last_became_active":"2024-09-13T14:49:59.507374+0000","last_became_peered":"2024-09-13T14:49:59.507374+0000","last_unstale":"2024-09-13T14:50:32.980704+0000","last_undegraded":"2024-09-13T14:50:32.980704+0000","last_fullsized":"2024-09-13T14:50:32.980704+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2874,"log_dups_size":0,"ondisk_log_size":2874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:22.009750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188350464,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2874,"num_write_kb":183936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529907+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:50:09.529907+0000","last_peered":"2024-09-13T14:50:09.529907+0000","last_clean":"2024-09-13T14:50:09.529907+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:50:09.529907+0000","last_undegraded":"2024-09-13T14:50:09.529907+0000","last_fullsized":"2024-09-13T14:50:09.529907+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":325,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529780+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:50:09.529780+0000","last_peered":"2024-09-13T14:50:09.529780+0000","last_clean":"2024-09-13T14:50:09.529780+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:50:09.529780+0000","last_undegraded":"2024-09-13T14:50:09.529780+0000","last_fullsized":"2024-09-13T14:50:09.529780+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1187,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744485+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:50:29.744485+0000","last_peered":"2024-09-13T14:50:29.744485+0000","last_clean":"2024-09-13T14:50:29.744485+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:50:29.744485+0000","last_undegraded":"2024-09-13T14:50:29.744485+0000","last_fullsized":"2024-09-13T14:50:29.744485+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":589,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745746+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:50:29.745746+0000","last_peered":"2024-09-13T14:50:29.745746+0000","last_clean":"2024-09-13T14:50:29.745746+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:50:29.745746+0000","last_undegraded":"2024-09-13T14:50:29.745746+0000","last_fullsized":"2024-09-13T14:50:29.745746+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"380'2900","reported_seq":2912,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.675772+0000","last_change":"2024-09-13T14:49:59.754749+0000","last_active":"2024-09-13T14:50:32.675772+0000","last_peered":"2024-09-13T14:50:32.675772+0000","last_clean":"2024-09-13T14:50:32.675772+0000","last_became_active":"2024-09-13T14:49:59.754398+0000","last_became_peered":"2024-09-13T14:49:59.754398+0000","last_unstale":"2024-09-13T14:50:32.675772+0000","last_undegraded":"2024-09-13T14:50:32.675772+0000","last_fullsized":"2024-09-13T14:50:32.675772+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2900,"log_dups_size":0,"ondisk_log_size":2900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:08.387473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190054400,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2900,"num_write_kb":185600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530519+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:50:09.530519+0000","last_peered":"2024-09-13T14:50:09.530519+0000","last_clean":"2024-09-13T14:50:09.530519+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:50:09.530519+0000","last_undegraded":"2024-09-13T14:50:09.530519+0000","last_fullsized":"2024-09-13T14:50:09.530519+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":598,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744380+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:50:29.744380+0000","last_peered":"2024-09-13T14:50:29.744380+0000","last_clean":"2024-09-13T14:50:29.744380+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:50:29.744380+0000","last_undegraded":"2024-09-13T14:50:29.744380+0000","last_fullsized":"2024-09-13T14:50:29.744380+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":774,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495956+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:50:10.495956+0000","last_peered":"2024-09-13T14:50:10.495956+0000","last_clean":"2024-09-13T14:50:10.495956+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:50:10.495956+0000","last_undegraded":"2024-09-13T14:50:10.495956+0000","last_fullsized":"2024-09-13T14:50:10.495956+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":631,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494825+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:50:10.494825+0000","last_peered":"2024-09-13T14:50:10.494825+0000","last_clean":"2024-09-13T14:50:10.494825+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:50:10.494825+0000","last_undegraded":"2024-09-13T14:50:10.494825+0000","last_fullsized":"2024-09-13T14:50:10.494825+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"380'2889","reported_seq":2901,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:32.675631+0000","last_change":"2024-09-13T14:49:59.511444+0000","last_active":"2024-09-13T14:50:32.675631+0000","last_peered":"2024-09-13T14:50:32.675631+0000","last_clean":"2024-09-13T14:50:32.675631+0000","last_became_active":"2024-09-13T14:49:59.511154+0000","last_became_peered":"2024-09-13T14:49:59.511154+0000","last_unstale":"2024-09-13T14:50:32.675631+0000","last_undegraded":"2024-09-13T14:50:32.675631+0000","last_fullsized":"2024-09-13T14:50:32.675631+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2889,"log_dups_size":0,"ondisk_log_size":2889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:18.382270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189333504,"num_objects":2889,"num_object_clones":0,"num_object_copies":8667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2889,"num_write_kb":184896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494777+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:50:10.494777+0000","last_peered":"2024-09-13T14:50:10.494777+0000","last_clean":"2024-09-13T14:50:10.494777+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:50:10.494777+0000","last_undegraded":"2024-09-13T14:50:10.494777+0000","last_fullsized":"2024-09-13T14:50:10.494777+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":634,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.741861+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:50:29.741861+0000","last_peered":"2024-09-13T14:50:29.741861+0000","last_clean":"2024-09-13T14:50:29.741861+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:50:29.741861+0000","last_undegraded":"2024-09-13T14:50:29.741861+0000","last_fullsized":"2024-09-13T14:50:29.741861+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":974,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.747414+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:50:29.747414+0000","last_peered":"2024-09-13T14:50:29.747414+0000","last_clean":"2024-09-13T14:50:29.747414+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:50:29.747414+0000","last_undegraded":"2024-09-13T14:50:29.747414+0000","last_fullsized":"2024-09-13T14:50:29.747414+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":632,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737711+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:50:29.737711+0000","last_peered":"2024-09-13T14:50:29.737711+0000","last_clean":"2024-09-13T14:50:29.737711+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:50:29.737711+0000","last_undegraded":"2024-09-13T14:50:29.737711+0000","last_fullsized":"2024-09-13T14:50:29.737711+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"380'2601","reported_seq":2620,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:30.879868+0000","last_change":"2024-09-13T14:49:59.755421+0000","last_active":"2024-09-13T14:50:30.879868+0000","last_peered":"2024-09-13T14:50:30.879868+0000","last_clean":"2024-09-13T14:50:30.879868+0000","last_became_active":"2024-09-13T14:49:59.755094+0000","last_became_peered":"2024-09-13T14:49:59.755094+0000","last_unstale":"2024-09-13T14:50:30.879868+0000","last_undegraded":"2024-09-13T14:50:30.879868+0000","last_fullsized":"2024-09-13T14:50:30.879868+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":2601,"log_dups_size":0,"ondisk_log_size":2601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:27.511205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2601,"num_write_kb":166464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":485,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737521+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:50:29.737521+0000","last_peered":"2024-09-13T14:50:29.737521+0000","last_clean":"2024-09-13T14:50:29.737521+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:50:29.737521+0000","last_undegraded":"2024-09-13T14:50:29.737521+0000","last_fullsized":"2024-09-13T14:50:29.737521+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":667,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495159+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:50:10.495159+0000","last_peered":"2024-09-13T14:50:10.495159+0000","last_clean":"2024-09-13T14:50:10.495159+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:50:10.495159+0000","last_undegraded":"2024-09-13T14:50:10.495159+0000","last_fullsized":"2024-09-13T14:50:10.495159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":798,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530072+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:50:09.530072+0000","last_peered":"2024-09-13T14:50:09.530072+0000","last_clean":"2024-09-13T14:50:09.530072+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:50:09.530072+0000","last_undegraded":"2024-09-13T14:50:09.530072+0000","last_fullsized":"2024-09-13T14:50:09.530072+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495721+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:50:10.495721+0000","last_peered":"2024-09-13T14:50:10.495721+0000","last_clean":"2024-09-13T14:50:10.495721+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:50:10.495721+0000","last_undegraded":"2024-09-13T14:50:10.495721+0000","last_fullsized":"2024-09-13T14:50:10.495721+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494751+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:50:10.494751+0000","last_peered":"2024-09-13T14:50:10.494751+0000","last_clean":"2024-09-13T14:50:10.494751+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:50:10.494751+0000","last_undegraded":"2024-09-13T14:50:10.494751+0000","last_fullsized":"2024-09-13T14:50:10.494751+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":89,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494868+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:50:10.494868+0000","last_peered":"2024-09-13T14:50:10.494868+0000","last_clean":"2024-09-13T14:50:10.494868+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:50:10.494868+0000","last_undegraded":"2024-09-13T14:50:10.494868+0000","last_fullsized":"2024-09-13T14:50:10.494868+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":585,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745622+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:50:29.745622+0000","last_peered":"2024-09-13T14:50:29.745622+0000","last_clean":"2024-09-13T14:50:29.745622+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:50:29.745622+0000","last_undegraded":"2024-09-13T14:50:29.745622+0000","last_fullsized":"2024-09-13T14:50:29.745622+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495727+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:50:10.495727+0000","last_peered":"2024-09-13T14:50:10.495727+0000","last_clean":"2024-09-13T14:50:10.495727+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:50:10.495727+0000","last_undegraded":"2024-09-13T14:50:10.495727+0000","last_fullsized":"2024-09-13T14:50:10.495727+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530774+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:50:09.530774+0000","last_peered":"2024-09-13T14:50:09.530774+0000","last_clean":"2024-09-13T14:50:09.530774+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:50:09.530774+0000","last_undegraded":"2024-09-13T14:50:09.530774+0000","last_fullsized":"2024-09-13T14:50:09.530774+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":590,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.736952+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:50:29.736952+0000","last_peered":"2024-09-13T14:50:29.736952+0000","last_clean":"2024-09-13T14:50:29.736952+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:50:29.736952+0000","last_undegraded":"2024-09-13T14:50:29.736952+0000","last_fullsized":"2024-09-13T14:50:29.736952+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":639,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495020+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:50:10.495020+0000","last_peered":"2024-09-13T14:50:10.495020+0000","last_clean":"2024-09-13T14:50:10.495020+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:50:10.495020+0000","last_undegraded":"2024-09-13T14:50:10.495020+0000","last_fullsized":"2024-09-13T14:50:10.495020+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":646,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737622+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:50:29.737622+0000","last_peered":"2024-09-13T14:50:29.737622+0000","last_clean":"2024-09-13T14:50:29.737622+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:50:29.737622+0000","last_undegraded":"2024-09-13T14:50:29.737622+0000","last_fullsized":"2024-09-13T14:50:29.737622+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529991+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:50:09.529991+0000","last_peered":"2024-09-13T14:50:09.529991+0000","last_clean":"2024-09-13T14:50:09.529991+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:50:09.529991+0000","last_undegraded":"2024-09-13T14:50:09.529991+0000","last_fullsized":"2024-09-13T14:50:09.529991+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":663,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.738204+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:50:29.738204+0000","last_peered":"2024-09-13T14:50:29.738204+0000","last_clean":"2024-09-13T14:50:29.738204+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:50:29.738204+0000","last_undegraded":"2024-09-13T14:50:29.738204+0000","last_fullsized":"2024-09-13T14:50:29.738204+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":632,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495447+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:50:10.495447+0000","last_peered":"2024-09-13T14:50:10.495447+0000","last_clean":"2024-09-13T14:50:10.495447+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:50:10.495447+0000","last_undegraded":"2024-09-13T14:50:10.495447+0000","last_fullsized":"2024-09-13T14:50:10.495447+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":610,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744066+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:50:29.744066+0000","last_peered":"2024-09-13T14:50:29.744066+0000","last_clean":"2024-09-13T14:50:29.744066+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:50:29.744066+0000","last_undegraded":"2024-09-13T14:50:29.744066+0000","last_fullsized":"2024-09-13T14:50:29.744066+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":659,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.736931+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:50:29.736931+0000","last_peered":"2024-09-13T14:50:29.736931+0000","last_clean":"2024-09-13T14:50:29.736931+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:50:29.736931+0000","last_undegraded":"2024-09-13T14:50:29.736931+0000","last_fullsized":"2024-09-13T14:50:29.736931+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":78,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495129+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:50:10.495129+0000","last_peered":"2024-09-13T14:50:10.495129+0000","last_clean":"2024-09-13T14:50:10.495129+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:50:10.495129+0000","last_undegraded":"2024-09-13T14:50:10.495129+0000","last_fullsized":"2024-09-13T14:50:10.495129+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":556,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744649+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:50:29.744649+0000","last_peered":"2024-09-13T14:50:29.744649+0000","last_clean":"2024-09-13T14:50:29.744649+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:50:29.744649+0000","last_undegraded":"2024-09-13T14:50:29.744649+0000","last_fullsized":"2024-09-13T14:50:29.744649+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":635,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494714+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:50:10.494714+0000","last_peered":"2024-09-13T14:50:10.494714+0000","last_clean":"2024-09-13T14:50:10.494714+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:50:10.494714+0000","last_undegraded":"2024-09-13T14:50:10.494714+0000","last_fullsized":"2024-09-13T14:50:10.494714+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495021+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:50:10.495021+0000","last_peered":"2024-09-13T14:50:10.495021+0000","last_clean":"2024-09-13T14:50:10.495021+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:50:10.495021+0000","last_undegraded":"2024-09-13T14:50:10.495021+0000","last_fullsized":"2024-09-13T14:50:10.495021+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529572+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:50:09.529572+0000","last_peered":"2024-09-13T14:50:09.529572+0000","last_clean":"2024-09-13T14:50:09.529572+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:50:09.529572+0000","last_undegraded":"2024-09-13T14:50:09.529572+0000","last_fullsized":"2024-09-13T14:50:09.529572+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495766+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:50:10.495766+0000","last_peered":"2024-09-13T14:50:10.495766+0000","last_clean":"2024-09-13T14:50:10.495766+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:50:10.495766+0000","last_undegraded":"2024-09-13T14:50:10.495766+0000","last_fullsized":"2024-09-13T14:50:10.495766+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":311,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530064+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:50:09.530064+0000","last_peered":"2024-09-13T14:50:09.530064+0000","last_clean":"2024-09-13T14:50:09.530064+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:50:09.530064+0000","last_undegraded":"2024-09-13T14:50:09.530064+0000","last_fullsized":"2024-09-13T14:50:09.530064+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":655,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737310+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:50:29.737310+0000","last_peered":"2024-09-13T14:50:29.737310+0000","last_clean":"2024-09-13T14:50:29.737310+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:50:29.737310+0000","last_undegraded":"2024-09-13T14:50:29.737310+0000","last_fullsized":"2024-09-13T14:50:29.737310+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":643,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.737390+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:50:29.737390+0000","last_peered":"2024-09-13T14:50:29.737390+0000","last_clean":"2024-09-13T14:50:29.737390+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:50:29.737390+0000","last_undegraded":"2024-09-13T14:50:29.737390+0000","last_fullsized":"2024-09-13T14:50:29.737390+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494619+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:50:10.494619+0000","last_peered":"2024-09-13T14:50:10.494619+0000","last_clean":"2024-09-13T14:50:10.494619+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:50:10.494619+0000","last_undegraded":"2024-09-13T14:50:10.494619+0000","last_fullsized":"2024-09-13T14:50:10.494619+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":645,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495016+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:50:10.495016+0000","last_peered":"2024-09-13T14:50:10.495016+0000","last_clean":"2024-09-13T14:50:10.495016+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:50:10.495016+0000","last_undegraded":"2024-09-13T14:50:10.495016+0000","last_fullsized":"2024-09-13T14:50:10.495016+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2822111232,"num_objects":43062,"num_object_clones":0,"num_object_copies":129186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43062,"num_write_kb":2755968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8575188992,"data_stored":8575188992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43062,"ondisk_log_size":43062,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397973,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2491812,"kb_used_data":1595800,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":91253340,"statfs":{"total":95995035648,"available":93443420160,"internally_reserved":0,"allocated":1634099200,"data_stored":1632052367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594201,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708240,"kb_used_data":801288,"kb_used_omap":1,"kb_used_meta":906942,"kb_avail":92036912,"statfs":{"total":95995035648,"available":94245797888,"internally_reserved":0,"allocated":820518912,"data_stored":818487761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":928708952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665655,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1971148,"kb_used_data":1634436,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":91774004,"statfs":{"total":95995035648,"available":93976580096,"internally_reserved":0,"allocated":1673662464,"data_stored":1671631716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986657,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1448988,"kb_used_data":553300,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92296164,"statfs":{"total":95995035648,"available":94511271936,"internally_reserved":0,"allocated":566579200,"data_stored":564536393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697216,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2466768,"kb_used_data":1618372,"kb_used_omap":4,"kb_used_meta":848379,"kb_avail":91278384,"statfs":{"total":95995035648,"available":93469065216,"internally_reserved":0,"allocated":1657212928,"data_stored":1655182261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":868740171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860743,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197328,"kb_used_data":522056,"kb_used_omap":8,"kb_used_meta":675255,"kb_avail":92547824,"statfs":{"total":95995035648,"available":94768971776,"internally_reserved":0,"allocated":534585344,"data_stored":532551095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":691461964},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020414,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1852364,"kb_used_data":1472448,"kb_used_omap":9,"kb_used_meta":379894,"kb_avail":91892788,"statfs":{"total":95995035648,"available":94098214912,"internally_reserved":0,"allocated":1507786752,"data_stored":1505759558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":389012293},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187804,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1273660,"kb_used_data":623540,"kb_used_omap":7,"kb_used_meta":650104,"kb_avail":92471492,"statfs":{"total":95995035648,"available":94690807808,"internally_reserved":0,"allocated":638504960,"data_stored":636481986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":665706827},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":640155648,"data_stored":640155648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1526988800,"data_stored":1526988800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":548470784,"data_stored":548470784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1381892096,"data_stored":1381892096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":565051392,"data_stored":565051392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1683881984,"data_stored":1683881984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":555810816,"data_stored":555810816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1672937472,"data_stored":1672937472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:50:41.931 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:50:41.932 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:50:41.932 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2024-09-13T14:50:41.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:50:41.933 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:41.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.960 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:42.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:42.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:42.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:42.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:42.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:42.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:42.509 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:42 smithi017 ceph-mon[103001]: pgmap v1276: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-13T14:50:42.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:42.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:42 smithi017 ceph-mon[107556]: pgmap v1276: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-13T14:50:42.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:42.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:42.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:42.724 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:42 smithi189 ceph-mon[90992]: pgmap v1276: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-09-13T14:50:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:42.802 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 61224 61208 127.494 88.6875 0.022268 0.00783788 2024-09-13T14:50:42.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:42.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:42.961 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:42.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:43.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:43.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:43.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:43.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89680 8.4 8:21ec7e63:::benchmark_data_smithi017_253885_object89679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:50:43.238 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 599.51480946900006, 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3691795170000001, 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89680 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716045+0000", 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716046+0000", 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3740000000000001e-06 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716071+0000", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4990999999999998e-05 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716082+0000", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1008e-05 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716090+0000", 2024-09-13T14:50:43.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9459999999999998e-06 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716141+0000", 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0575999999999998e-05 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.716151+0000", 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8810000000000008e-06 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699865+0000", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98371478300000004 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699883+0000", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8005000000000001e-05 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057337+0000", 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3574534739999999 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057492+0000", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015483399999999999 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.057629+0000", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137315 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064142+0000", 2024-09-13T14:50:43.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006512838 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.077471+0000", 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013329314 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.081522+0000", 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040507049999999999 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.081591+0000", 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9319999999999994e-05 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085224+0000", 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036331530000000001 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89696 8.4 8:26ba4658:::benchmark_data_smithi017_253885_object89695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e316)", 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:50:43.246 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 599.49017970199998, 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3480544769999998, 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89696 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:50:43.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740675+0000", 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740676+0000", 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3039999999999999e-06 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740696+0000", 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9915000000000002e-05 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740708+0000", 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.222e-05 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740716+0000", 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.807e-06 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740785+0000", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9041999999999994e-05 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:43.740798+0000", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2714999999999999e-05 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699915+0000", 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95911772799999995 2024-09-13T14:50:43.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.699925+0000", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8910000000000007e-06 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064178+0000", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.36425273 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064355+0000", 2024-09-13T14:50:43.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177307 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.064542+0000", 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000186985 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.067290+0000", 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00274727 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.077518+0000", 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010228331 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085295+0000", 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077774150000000002 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.085371+0000", 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5306e-05 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.088729+0000", 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033585109999999998 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89697 8.d 8:bb121d82:::benchmark_data_smithi017_253885_object89696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 598.53777693400002, 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.382211367, 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.256 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89697 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693077+0000", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693080+0000", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.362e-06 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693112+0000", 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2036999999999998e-05 2024-09-13T14:50:43.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693137+0000", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4749999999999999e-05 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693148+0000", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1097999999999999e-05 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693757+0000", 2024-09-13T14:50:43.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060958799999999999 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693765+0000", 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2629999999999997e-06 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.697968+0000", 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042034730000000001 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.697977+0000", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4450000000000002e-06 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.050416+0000", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3524383870000001 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.053361+0000", 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029451220000000001 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.053511+0000", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000149949 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062676+0000", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091654820000000008 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062800+0000", 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012333600000000001 2024-09-13T14:50:43.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.073014+0000", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010214076000000001 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.073062+0000", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8010999999999998e-05 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075289+0000", 2024-09-13T14:50:43.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022269880000000001 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:89699 8.d 8:b4b5c04d:::benchmark_data_smithi017_253885_object89698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 598.53767394600004, 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3850268240000001, 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.264 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 89699 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693180+0000", 2024-09-13T14:50:43.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693181+0000", 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.99e-07 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693207+0000", 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6259e-05 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693216+0000", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9760000000000001e-06 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693223+0000", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9829999999999999e-06 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693816+0000", 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00059274000000000002 2024-09-13T14:50:43.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.693821+0000", 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7010000000000004e-06 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.698001+0000", 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041799910000000001 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:44.698008+0000", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8900000000000001e-06 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059361+0000", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.361353665 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059552+0000", 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019093800000000001 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.059715+0000", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016238900000000001 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.062807+0000", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030924160000000002 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075362+0000", 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012554872 2024-09-13T14:50:43.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075460+0000", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8325000000000005e-05 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.075525+0000", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4801000000000005e-05 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:40:46.078207+0000", 2024-09-13T14:50:43.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026822790000000001 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:127568 8.4 8:2cc62b5e:::benchmark_data_smithi017_253885_object36227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.01710355799997, 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17335995600000001, 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.272 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 127568 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213753+0000", 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0940000000000002e-06 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213760+0000", 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3869999999999997e-06 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213777+0000", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6974e-05 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213787+0000", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5759999999999995e-06 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213845+0000", 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7785000000000003e-05 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.214225+0000", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038041600000000001 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220181+0000", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059560539999999997 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220236+0000", 2024-09-13T14:50:43.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5072999999999999e-05 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220317+0000", 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0638999999999998e-05 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.224135+0000", 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038185010000000002 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.224973+0000", 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00083726000000000004 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226407+0000", 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001434664 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226447+0000", 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9643e-05 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.387111+0000", 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16066389 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:127568 8.4 e355/317 ondisk, result = 0)", 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.00450882799998, 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16077082700000001, 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.279 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5400000000000002e-07 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226349+0000", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.033e-06 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226356+0000", 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5039999999999999e-06 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226359+0000", 2024-09-13T14:50:43.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6990000000000001e-06 2024-09-13T14:50:43.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226398+0000", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8414999999999997e-05 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226403+0000", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.57e-06 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.387116+0000", 2024-09-13T14:50:43.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.160713152 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:127569 8.1 e355/303 8:847fb8fd:::benchmark_data_smithi017_253885_object36228:head v 355'7703, mlcod=355'7701)", 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.003819201, 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16228176999999999, 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.284 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227036+0000", 2024-09-13T14:50:43.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8700000000000005e-07 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227043+0000", 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8519999999999998e-06 2024-09-13T14:50:43.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385983+0000", 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15894031 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385993+0000", 2024-09-13T14:50:43.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8260000000000005e-06 2024-09-13T14:50:43.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386049+0000", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5847999999999998e-05 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386073+0000", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4431999999999999e-05 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389281+0000", 2024-09-13T14:50:43.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032075879999999999 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389317+0000", 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6127000000000002e-05 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:127576 8.e e355/337 8:7d734908:::benchmark_data_smithi017_253885_object36235:head v 355'7935, mlcod=355'7933)", 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.330811+0000", 2024-09-13T14:50:43.292 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 287.90004364499998, 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.059128879000000002, 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330811+0000", 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330811+0000", 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330812+0000", 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.207e-06 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.330821+0000", 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7069999999999998e-06 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385979+0000", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.055158253999999997 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385992+0000", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3499999999999999e-05 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386068+0000", 2024-09-13T14:50:43.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6049000000000003e-05 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386088+0000", 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9707e-05 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389907+0000", 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038188609999999998 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389940+0000", 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2594e-05 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:143138 8.4 8:27c3d854:::benchmark_data_smithi017_253885_object51797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:47:28.673335+0000", 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 194.55751942699999, 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.044970866999999998, 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.297 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 143138 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673335+0000", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673335+0000", 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673336+0000", 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.314e-06 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673343+0000", 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.268e-06 2024-09-13T14:50:43.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673355+0000", 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2361000000000001e-05 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.673361+0000", 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1720000000000004e-06 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.678976+0000", 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056146499999999997 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.679554+0000", 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00057860300000000002 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.694156+0000", 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014601873 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.694220+0000", 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3360000000000003e-05 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.694312+0000", 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2875999999999996e-05 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.712314+0000", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018001968 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.716277+0000", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039623610000000002 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.716340+0000", 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3193000000000003e-05 2024-09-13T14:50:43.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.716385+0000", 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5009999999999998e-05 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:47:28.718306+0000", 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019208579999999999 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:155993 8.4 8:2bff1898:::benchmark_data_smithi017_253885_object64652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:24.352327+0000", 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 138.87852741699999, 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.043580953999999998, 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 155993 2024-09-13T14:50:43.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352327+0000", 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352327+0000", 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352329+0000", 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9470000000000002e-06 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352335+0000", 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0329999999999997e-06 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352348+0000", 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2767e-05 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.352354+0000", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0209999999999996e-06 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.362737+0000", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010382874 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.363266+0000", 2024-09-13T14:50:43.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052893000000000005 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.375263+0000", 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011997582 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.375390+0000", 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012657900000000001 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.375565+0000", 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017569100000000001 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.381324+0000", 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057585589999999999 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.381336+0000", 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1555e-05 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.394598+0000", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013262723000000001 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.394669+0000", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0481999999999994e-05 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.395908+0000", 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001239211 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.311 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:155994 8.4 8:265e53c1:::benchmark_data_smithi017_253885_object64653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:24.354794+0000", 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 138.87606, 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042633451000000003, 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.312 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 155994 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354794+0000", 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354794+0000", 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354795+0000", 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.099e-06 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354800+0000", 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4569999999999998e-06 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354813+0000", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3344e-05 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.354820+0000", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6880000000000002e-06 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.363303+0000", 2024-09-13T14:50:43.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084829109999999992 2024-09-13T14:50:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:43.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:43.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.363769+0000", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046619800000000002 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.383557+0000", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019787705999999999 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.383668+0000", 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011095 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.383776+0000", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010787 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.395972+0000", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012195921 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.395989+0000", 2024-09-13T14:50:43.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7850999999999999e-05 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.396078+0000", 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9069999999999999e-05 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.396137+0000", 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8189000000000001e-05 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:24.397428+0000", 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001291197 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:166879 8.4 8:2cc6d762:::benchmark_data_smithi017_253885_object75538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:57.442548+0000", 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 105.788306737, 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.043849022000000001, 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166879 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442548+0000", 2024-09-13T14:50:43.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442548+0000", 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442549+0000", 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0920000000000001e-06 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442553+0000", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6809999999999997e-06 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442564+0000", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0968000000000001e-05 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.442572+0000", 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8339999999999999e-06 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.448435+0000", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058627719999999996 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.448578+0000", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000142893 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.448726+0000", 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014799 2024-09-13T14:50:43.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.457645+0000", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089186979999999992 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.478925+0000", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021280688999999998 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.485503+0000", 2024-09-13T14:50:43.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065775859999999998 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.485569+0000", 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5956000000000003e-05 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:57.486397+0000", 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082786299999999997 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.328 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:167635 8.4 8:20d62d3a:::benchmark_data_smithi017_253885_object76294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:58.478956+0000", 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.751898641, 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.046445063000000002, 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.329 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 167635 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478956+0000", 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478956+0000", 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478957+0000", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5799999999999998e-07 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478961+0000", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0600000000000001e-06 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478976+0000", 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4972e-05 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.478982+0000", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4189999999999999e-06 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482936+0000", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039537330000000001 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.483085+0000", 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000149104 2024-09-13T14:50:43.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.483229+0000", 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144086 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.493862+0000", 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010633425 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.498135+0000", 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042724950000000003 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.524722+0000", 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026587415 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.524762+0000", 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9805000000000003e-05 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.525401+0000", 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00063869100000000004 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:167638 8.4 8:210b7f3e:::benchmark_data_smithi017_253885_object76297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:58.482591+0000", 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.748263874, 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.043484047999999997, 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.335 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 167638 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482591+0000", 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482591+0000", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482592+0000", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9750000000000001e-06 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482599+0000", 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3040000000000001e-06 2024-09-13T14:50:43.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482621+0000", 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1934999999999999e-05 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.482630+0000", 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9940000000000006e-06 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.487960+0000", 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053305719999999996 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.488063+0000", 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102484 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.488153+0000", 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9838000000000003e-05 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.493870+0000", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057175680000000001 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.498187+0000", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043171829999999996 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.525468+0000", 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027280332000000001 2024-09-13T14:50:43.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.525535+0000", 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7286999999999995e-05 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.526075+0000", 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00053957600000000005 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:167751 8.4 8:2e594044:::benchmark_data_smithi017_253885_object76410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:58.650208+0000", 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.580646452, 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042132102999999997, 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 167751 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.342 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650208+0000", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650208+0000", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650210+0000", 2024-09-13T14:50:43.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.618e-06 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650215+0000", 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0599999999999998e-06 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650227+0000", 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2139e-05 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650234+0000", 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5719999999999997e-06 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650713+0000", 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047915700000000002 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650858+0000", 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014444800000000001 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.651023+0000", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016547900000000001 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.673292+0000", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.02226887 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.680509+0000", 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072171079999999999 2024-09-13T14:50:43.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.691427+0000", 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010917937000000001 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.691486+0000", 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9039000000000002e-05 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.692340+0000", 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085367600000000004 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:167755 8.4 8:26d14246:::benchmark_data_smithi017_253885_object76414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:48:58.650248+0000", 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.580606014, 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042679086999999997, 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.348 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 167755 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650248+0000", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650248+0000", 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650249+0000", 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1800000000000001e-07 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650251+0000", 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.013e-06 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650257+0000", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0079999999999999e-06 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.650262+0000", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7439999999999998e-06 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.657784+0000", 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075222370000000002 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.657961+0000", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017757299999999999 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.658138+0000", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017698799999999999 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.673301+0000", 2024-09-13T14:50:43.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015162366 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.680615+0000", 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073141220000000002 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.692391+0000", 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011776485999999999 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.692528+0000", 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136563 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:48:58.692927+0000", 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039956900000000001 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:168892 8.4 8:26ec9a98:::benchmark_data_smithi017_253885_object77551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:49:00.157270+0000", 2024-09-13T14:50:43.354 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 103.073584324, 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.043976552000000002, 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 168892 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157270+0000", 2024-09-13T14:50:43.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157270+0000", 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157272+0000", 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.559e-06 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157277+0000", 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4820000000000002e-06 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157290+0000", 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2583e-05 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.157297+0000", 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7940000000000003e-06 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.162365+0000", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050673030000000004 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.162813+0000", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044823699999999998 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.163125+0000", 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031170999999999998 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.180670+0000", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017545694000000001 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.186749+0000", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060783030000000002 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.200396+0000", 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013646861 2024-09-13T14:50:43.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.200459+0000", 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3860000000000002e-05 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.201247+0000", 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078716599999999995 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.361 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:168895 8.4 8:29d9b59e:::benchmark_data_smithi017_253885_object77554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:50:43.361 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:49:00.160266+0000", 2024-09-13T14:50:43.361 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 103.070588156, 2024-09-13T14:50:43.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041673413999999999, 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 168895 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160266+0000", 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160266+0000", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160268+0000", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9130000000000001e-06 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160274+0000", 2024-09-13T14:50:43.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2299999999999996e-06 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160296+0000", 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1832e-05 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.160306+0000", 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4720000000000001e-06 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.168643+0000", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083375689999999995 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.168813+0000", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016966799999999999 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.168980+0000", 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166992 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.180703+0000", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011723416 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.186998+0000", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062950030000000004 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.201299+0000", 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014300475 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.201398+0000", 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8902999999999998e-05 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:00.201940+0000", 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054194099999999999 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:181730 8.4 8:2105b34c:::benchmark_data_smithi017_253885_object90389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:49:17.247871+0000", 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 85.982983111999999, 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.049309533000000003, 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 181730 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247871+0000", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247871+0000", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247873+0000", 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.407e-06 2024-09-13T14:50:43.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247881+0000", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1759999999999993e-06 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247890+0000", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6199999999999994e-06 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.247896+0000", 2024-09-13T14:50:43.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0070000000000003e-06 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.256280+0000", 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008383125 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.256460+0000", 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018042000000000001 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.256677+0000", 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000217066 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.271629+0000", 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014952208999999999 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.282162+0000", 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010532501999999999 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.296401+0000", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014239628000000001 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.296568+0000", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016697700000000001 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.297181+0000", 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00061239599999999997 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.373 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:181732 8.4 8:2c64ff86:::benchmark_data_smithi017_253885_object90391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:49:17.255708+0000", 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 85.975146312000007, 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041923848, 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:50:43.374 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 181732 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255708+0000", 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255708+0000", 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255710+0000", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6869999999999999e-06 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255716+0000", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3720000000000001e-06 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255729+0000", 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2683999999999999e-05 2024-09-13T14:50:43.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.255736+0000", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2740000000000002e-06 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.263339+0000", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076026829999999998 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.263609+0000", 2024-09-13T14:50:43.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027007099999999998 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.264050+0000", 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044101600000000002 2024-09-13T14:50:43.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.271642+0000", 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075923639999999999 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.282324+0000", 2024-09-13T14:50:43.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010681763 2024-09-13T14:50:43.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.297247+0000", 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014923087999999999 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.297357+0000", 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109695 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:50:43.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:49:17.297632+0000", 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027515100000000001 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.382 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.383 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:50:43.383 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:50:43.383 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:50:43.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:43.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:43.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:43.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:43.696 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:43.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:43.803 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 62738 62722 126.433 94.625 0.0118444 0.00790422 2024-09-13T14:50:43.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:43.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:44.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:44.400 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:44.536 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:44.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:44 smithi017 ceph-mon[103001]: pgmap v1277: 99 pgs: 99 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:44.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:44 smithi017 ceph-mon[107556]: pgmap v1277: 99 pgs: 99 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:44.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:44.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:44 smithi189 ceph-mon[90992]: pgmap v1277: 99 pgs: 99 active+clean; 3.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:44.803 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 64288 64272 125.509 96.875 0.0100189 0.0079626 2024-09-13T14:50:44.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:44.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:44.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:44.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:44.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:45.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:45.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:45.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:45.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:45.362 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:45.390 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:45.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:45.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:45.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:45.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:45.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:45.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:45.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:45.804 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 15 65814 65799 124.592 95.4375 0.00924086 0.00802132 2024-09-13T14:50:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:45.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:45.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:50:45.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:45.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:46.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:46.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:46.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:46.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:46.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:46.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:46.430 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:46.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:46.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:46.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:46 smithi017 ceph-mon[103001]: pgmap v1278: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:50:46.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:46 smithi017 ceph-mon[107556]: pgmap v1278: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:50:46.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:46.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:46 smithi189 ceph-mon[90992]: pgmap v1278: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:50:46.804 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 67192 67176 123.459 86.0625 0.0131308 0.00809432 2024-09-13T14:50:46.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:46.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:47.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:47.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:47.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:47.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:47.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:47] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-09-13T14:50:47.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:47.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:47.634 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:47.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:47.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:47.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:50:47.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:50:47.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:47.740 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:47.741 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:47.760 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:50:47.761 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:47.806 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 68533 68517 122.324 83.8125 0.0146685 0.00816967 2024-09-13T14:50:47.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:47.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:47.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:47.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:47.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:48.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:48.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:48.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:48.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:48 smithi017 ceph-mon[103001]: pgmap v1279: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-13T14:50:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3868704306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:48 smithi017 ceph-mon[107556]: pgmap v1279: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-13T14:50:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3868704306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:48.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:48.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:48.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:48.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:48.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:48 smithi189 ceph-mon[90992]: pgmap v1279: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2024-09-13T14:50:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3868704306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:48.805 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 69979 69963 121.436 90.375 0.0134881 0.00822968 2024-09-13T14:50:48.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:48.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:48.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:48.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:49.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:49.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:49.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:49.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:49.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:49.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:49.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:49.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:49.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:49.777 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.805 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 71276 71260 120.345 81.0625 0.0155686 0.00830427 2024-09-13T14:50:49.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:49.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:50.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:50.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:50.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:50.408 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:50:50.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[103001]: pgmap v1280: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:50.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1878940955' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[107556]: pgmap v1280: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:50.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1878940955' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:50 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:50.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:50:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:50:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:50 smithi189 ceph-mon[90992]: pgmap v1280: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:50:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1878940955' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:50 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:50.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.735 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.806 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 72684 72668 119.494 88 0.00885461 0.00836355 2024-09-13T14:50:50.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:50.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:50.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:51.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:51.120 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:51.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:51.391 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:51.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:51.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:51.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:50:51.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:51.483 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[103001]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:50:51.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:51.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:50:51.484 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:51.485 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:51 smithi017 ceph-mon[107556]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:50:51.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:51.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:51.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:51.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:51 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:50:51.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:51 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:50:51.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:51 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:51.757 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:51 smithi189 ceph-mon[90992]: osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:50:51.778 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:51.778 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:51.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:51.805 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 74062 74046 118.638 86.125 0.0121618 0.00842413 2024-09-13T14:50:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:51.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:51.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:51.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:52.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.121 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:52.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:52.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:52.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:52.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:52.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:52.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:52.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:50:52.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:52.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:52 smithi017 ceph-mon[103001]: pgmap v1282: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:52 smithi017 ceph-mon[107556]: pgmap v1282: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:52.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:52 smithi189 ceph-mon[90992]: pgmap v1282: 99 pgs: 99 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:50:52.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:52.808 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:50:52.807800+0000 min lat: 0.00120371 max lat: 0.172172 avg lat: 0.00848124 2024-09-13T14:50:52.808 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:50:52.808 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 75446 75430 117.828 86.5 0.0114568 0.00848124 2024-09-13T14:50:52.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:52.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:52.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:52.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:53.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:53.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:53.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.277 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:50:53.278 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:50:53.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:53.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:53.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:53.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:53.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:53.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:53.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.808 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 76728 76712 116.907 80.125 0.00705706 0.00854906 2024-09-13T14:50:53.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:53.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:54.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:54.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:54.198 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1282,"stamp":"2024-09-13T14:50:51.324813+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4622522317,"num_objects":70489,"num_object_clones":0,"num_object_copies":211467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70489,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":71368,"num_write_kb":4522127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72593,"ondisk_log_size":72593,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19500884,"kb_used_data":13894160,"kb_used_omap":44,"kb_used_meta":5606611,"kb_avail":730460332,"statfs":{"total":767960285184,"available":747991379968,"internally_reserved":0,"allocated":14227619840,"data_stored":14211345309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":5741169911},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,5,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":954269696,"num_objects":14561,"num_object_clones":0,"num_object_copies":43683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14561,"num_write_kb":931904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003308"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":648,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.411275+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:50:50.411275+0000","last_peered":"2024-09-13T14:50:50.411275+0000","last_clean":"2024-09-13T14:50:50.411275+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:50:50.411275+0000","last_undegraded":"2024-09-13T14:50:50.411275+0000","last_fullsized":"2024-09-13T14:50:50.411275+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":378,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495076+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:50:10.495076+0000","last_peered":"2024-09-13T14:50:10.495076+0000","last_clean":"2024-09-13T14:50:10.495076+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:50:10.495076+0000","last_undegraded":"2024-09-13T14:50:10.495076+0000","last_fullsized":"2024-09-13T14:50:10.495076+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529983+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:50:09.529983+0000","last_peered":"2024-09-13T14:50:09.529983+0000","last_clean":"2024-09-13T14:50:09.529983+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:50:09.529983+0000","last_undegraded":"2024-09-13T14:50:09.529983+0000","last_fullsized":"2024-09-13T14:50:09.529983+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":631,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.740798+0000","last_change":"2024-09-13T14:24:00.654937+0000","last_active":"2024-09-13T14:50:29.740798+0000","last_peered":"2024-09-13T14:50:29.740798+0000","last_clean":"2024-09-13T14:50:29.740798+0000","last_became_active":"2024-09-13T14:24:00.654630+0000","last_became_peered":"2024-09-13T14:24:00.654630+0000","last_unstale":"2024-09-13T14:50:29.740798+0000","last_undegraded":"2024-09-13T14:50:29.740798+0000","last_fullsized":"2024-09-13T14:50:29.740798+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:43.557768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":632,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739562+0000","last_change":"2024-09-13T14:24:00.654463+0000","last_active":"2024-09-13T14:50:29.739562+0000","last_peered":"2024-09-13T14:50:29.739562+0000","last_clean":"2024-09-13T14:50:29.739562+0000","last_became_active":"2024-09-13T14:24:00.654162+0000","last_became_peered":"2024-09-13T14:24:00.654162+0000","last_unstale":"2024-09-13T14:50:29.739562+0000","last_undegraded":"2024-09-13T14:50:29.739562+0000","last_fullsized":"2024-09-13T14:50:29.739562+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:35.020532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494628+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:50:10.494628+0000","last_peered":"2024-09-13T14:50:10.494628+0000","last_clean":"2024-09-13T14:50:10.494628+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:50:10.494628+0000","last_undegraded":"2024-09-13T14:50:10.494628+0000","last_fullsized":"2024-09-13T14:50:10.494628+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":648,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407125+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:50:50.407125+0000","last_peered":"2024-09-13T14:50:50.407125+0000","last_clean":"2024-09-13T14:50:50.407125+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:50:50.407125+0000","last_undegraded":"2024-09-13T14:50:50.407125+0000","last_fullsized":"2024-09-13T14:50:50.407125+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":369,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529522+0000","last_change":"2024-09-13T14:40:44.705624+0000","last_active":"2024-09-13T14:50:09.529522+0000","last_peered":"2024-09-13T14:50:09.529522+0000","last_clean":"2024-09-13T14:50:09.529522+0000","last_became_active":"2024-09-13T14:40:44.705281+0000","last_became_peered":"2024-09-13T14:40:44.705281+0000","last_unstale":"2024-09-13T14:50:09.529522+0000","last_undegraded":"2024-09-13T14:50:09.529522+0000","last_fullsized":"2024-09-13T14:50:09.529522+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:35.939755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"380'4416","reported_seq":4435,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:49.116125+0000","last_change":"2024-09-13T14:49:59.509935+0000","last_active":"2024-09-13T14:50:49.116125+0000","last_peered":"2024-09-13T14:50:49.116125+0000","last_clean":"2024-09-13T14:50:49.116125+0000","last_became_active":"2024-09-13T14:49:59.509619+0000","last_became_peered":"2024-09-13T14:49:59.509619+0000","last_unstale":"2024-09-13T14:50:49.116125+0000","last_undegraded":"2024-09-13T14:50:49.116125+0000","last_fullsized":"2024-09-13T14:50:49.116125+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4416,"log_dups_size":0,"ondisk_log_size":4416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:39:57.083056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4416,"num_write_kb":282624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":373,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.496206+0000","last_change":"2024-09-13T14:40:26.275872+0000","last_active":"2024-09-13T14:50:10.496206+0000","last_peered":"2024-09-13T14:50:10.496206+0000","last_clean":"2024-09-13T14:50:10.496206+0000","last_became_active":"2024-09-13T14:40:26.258737+0000","last_became_peered":"2024-09-13T14:40:26.258737+0000","last_unstale":"2024-09-13T14:50:10.496206+0000","last_undegraded":"2024-09-13T14:50:10.496206+0000","last_fullsized":"2024-09-13T14:50:10.496206+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:54.514845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":292,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529198+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:50:09.529198+0000","last_peered":"2024-09-13T14:50:09.529198+0000","last_clean":"2024-09-13T14:50:09.529198+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:50:09.529198+0000","last_undegraded":"2024-09-13T14:50:09.529198+0000","last_fullsized":"2024-09-13T14:50:09.529198+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"380'4571","reported_seq":4590,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.179664+0000","last_change":"2024-09-13T14:49:59.755413+0000","last_active":"2024-09-13T14:50:50.179664+0000","last_peered":"2024-09-13T14:50:50.179664+0000","last_clean":"2024-09-13T14:50:50.179664+0000","last_became_active":"2024-09-13T14:49:59.755054+0000","last_became_peered":"2024-09-13T14:49:59.755054+0000","last_unstale":"2024-09-13T14:50:50.179664+0000","last_undegraded":"2024-09-13T14:50:50.179664+0000","last_fullsized":"2024-09-13T14:50:50.179664+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:10:24.559415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4571,"num_write_kb":292544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":391,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495228+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:50:10.495228+0000","last_peered":"2024-09-13T14:50:10.495228+0000","last_clean":"2024-09-13T14:50:10.495228+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:50:10.495228+0000","last_undegraded":"2024-09-13T14:50:10.495228+0000","last_fullsized":"2024-09-13T14:50:10.495228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494956+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:50:10.494956+0000","last_peered":"2024-09-13T14:50:10.494956+0000","last_clean":"2024-09-13T14:50:10.494956+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:50:10.494956+0000","last_undegraded":"2024-09-13T14:50:10.494956+0000","last_fullsized":"2024-09-13T14:50:10.494956+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"380'4183","reported_seq":4195,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.973330+0000","last_change":"2024-09-13T14:49:59.509888+0000","last_active":"2024-09-13T14:50:47.973330+0000","last_peered":"2024-09-13T14:50:47.973330+0000","last_clean":"2024-09-13T14:50:47.973330+0000","last_became_active":"2024-09-13T14:49:59.509562+0000","last_became_peered":"2024-09-13T14:49:59.509562+0000","last_unstale":"2024-09-13T14:50:47.973330+0000","last_undegraded":"2024-09-13T14:50:47.973330+0000","last_fullsized":"2024-09-13T14:50:47.973330+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4183,"log_dups_size":0,"ondisk_log_size":4183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:53.170631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274137088,"num_objects":4183,"num_object_clones":0,"num_object_copies":12549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4183,"num_write_kb":267712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495843+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:50:10.495843+0000","last_peered":"2024-09-13T14:50:10.495843+0000","last_clean":"2024-09-13T14:50:10.495843+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:50:10.495843+0000","last_undegraded":"2024-09-13T14:50:10.495843+0000","last_fullsized":"2024-09-13T14:50:10.495843+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":360,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529729+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:50:09.529729+0000","last_peered":"2024-09-13T14:50:09.529729+0000","last_clean":"2024-09-13T14:50:09.529729+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:50:09.529729+0000","last_undegraded":"2024-09-13T14:50:09.529729+0000","last_fullsized":"2024-09-13T14:50:09.529729+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"380'4331","reported_seq":4343,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.997485+0000","last_change":"2024-09-13T14:50:00.002234+0000","last_active":"2024-09-13T14:50:47.997485+0000","last_peered":"2024-09-13T14:50:47.997485+0000","last_clean":"2024-09-13T14:50:47.997485+0000","last_became_active":"2024-09-13T14:50:00.001900+0000","last_became_peered":"2024-09-13T14:50:00.001900+0000","last_unstale":"2024-09-13T14:50:47.997485+0000","last_undegraded":"2024-09-13T14:50:47.997485+0000","last_fullsized":"2024-09-13T14:50:47.997485+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4331,"log_dups_size":0,"ondisk_log_size":4331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:03.241074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283836416,"num_objects":4331,"num_object_clones":0,"num_object_copies":12993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4331,"num_write_kb":277184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529486+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:50:09.529486+0000","last_peered":"2024-09-13T14:50:09.529486+0000","last_clean":"2024-09-13T14:50:09.529486+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:50:09.529486+0000","last_undegraded":"2024-09-13T14:50:09.529486+0000","last_fullsized":"2024-09-13T14:50:09.529486+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":657,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739906+0000","last_change":"2024-09-13T14:40:26.269951+0000","last_active":"2024-09-13T14:50:29.739906+0000","last_peered":"2024-09-13T14:50:29.739906+0000","last_clean":"2024-09-13T14:50:29.739906+0000","last_became_active":"2024-09-13T14:40:26.269566+0000","last_became_peered":"2024-09-13T14:40:26.269566+0000","last_unstale":"2024-09-13T14:50:29.739906+0000","last_undegraded":"2024-09-13T14:50:29.739906+0000","last_fullsized":"2024-09-13T14:50:29.739906+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:49.775535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"381'4571","reported_seq":4592,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.878581+0000","last_change":"2024-09-13T14:49:59.511081+0000","last_active":"2024-09-13T14:50:50.878581+0000","last_peered":"2024-09-13T14:50:50.878581+0000","last_clean":"2024-09-13T14:50:50.878581+0000","last_became_active":"2024-09-13T14:49:59.510738+0000","last_became_peered":"2024-09-13T14:49:59.510738+0000","last_unstale":"2024-09-13T14:50:50.878581+0000","last_undegraded":"2024-09-13T14:50:50.878581+0000","last_fullsized":"2024-09-13T14:50:50.878581+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:53.029264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4571,"num_write_kb":292544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":491,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494923+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:50:10.494923+0000","last_peered":"2024-09-13T14:50:10.494923+0000","last_clean":"2024-09-13T14:50:10.494923+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:50:10.494923+0000","last_undegraded":"2024-09-13T14:50:10.494923+0000","last_fullsized":"2024-09-13T14:50:10.494923+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":657,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.406484+0000","last_change":"2024-09-13T14:40:26.261979+0000","last_active":"2024-09-13T14:50:50.406484+0000","last_peered":"2024-09-13T14:50:50.406484+0000","last_clean":"2024-09-13T14:50:50.406484+0000","last_became_active":"2024-09-13T14:40:26.255512+0000","last_became_peered":"2024-09-13T14:40:26.255512+0000","last_unstale":"2024-09-13T14:50:50.406484+0000","last_undegraded":"2024-09-13T14:50:50.406484+0000","last_fullsized":"2024-09-13T14:50:50.406484+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:14.819577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494715+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:50:10.494715+0000","last_peered":"2024-09-13T14:50:10.494715+0000","last_clean":"2024-09-13T14:50:10.494715+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:50:10.494715+0000","last_undegraded":"2024-09-13T14:50:10.494715+0000","last_fullsized":"2024-09-13T14:50:10.494715+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"380'4424","reported_seq":4442,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.184163+0000","last_change":"2024-09-13T14:50:00.002352+0000","last_active":"2024-09-13T14:50:50.184163+0000","last_peered":"2024-09-13T14:50:50.184163+0000","last_clean":"2024-09-13T14:50:50.184163+0000","last_became_active":"2024-09-13T14:50:00.002036+0000","last_became_peered":"2024-09-13T14:50:00.002036+0000","last_unstale":"2024-09-13T14:50:50.184163+0000","last_undegraded":"2024-09-13T14:50:50.184163+0000","last_fullsized":"2024-09-13T14:50:50.184163+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:29:00.760558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4424,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":205,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529314+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:50:09.529314+0000","last_peered":"2024-09-13T14:50:09.529314+0000","last_clean":"2024-09-13T14:50:09.529314+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:50:09.529314+0000","last_undegraded":"2024-09-13T14:50:09.529314+0000","last_fullsized":"2024-09-13T14:50:09.529314+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":381,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529592+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:50:09.529592+0000","last_peered":"2024-09-13T14:50:09.529592+0000","last_clean":"2024-09-13T14:50:09.529592+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:50:09.529592+0000","last_undegraded":"2024-09-13T14:50:09.529592+0000","last_fullsized":"2024-09-13T14:50:09.529592+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":93,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.496240+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:50:10.496240+0000","last_peered":"2024-09-13T14:50:10.496240+0000","last_clean":"2024-09-13T14:50:10.496240+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:50:10.496240+0000","last_undegraded":"2024-09-13T14:50:10.496240+0000","last_fullsized":"2024-09-13T14:50:10.496240+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"380'4344","reported_seq":4356,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.686415+0000","last_change":"2024-09-13T14:49:59.511648+0000","last_active":"2024-09-13T14:50:47.686415+0000","last_peered":"2024-09-13T14:50:47.686415+0000","last_clean":"2024-09-13T14:50:47.686415+0000","last_became_active":"2024-09-13T14:49:59.511315+0000","last_became_peered":"2024-09-13T14:49:59.511315+0000","last_unstale":"2024-09-13T14:50:47.686415+0000","last_undegraded":"2024-09-13T14:50:47.686415+0000","last_fullsized":"2024-09-13T14:50:47.686415+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4344,"log_dups_size":0,"ondisk_log_size":4344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:43.509125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284688384,"num_objects":4344,"num_object_clones":0,"num_object_copies":13032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4344,"num_write_kb":278016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495605+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:50:10.495605+0000","last_peered":"2024-09-13T14:50:10.495605+0000","last_clean":"2024-09-13T14:50:10.495605+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:50:10.495605+0000","last_undegraded":"2024-09-13T14:50:10.495605+0000","last_fullsized":"2024-09-13T14:50:10.495605+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495247+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:50:10.495247+0000","last_peered":"2024-09-13T14:50:10.495247+0000","last_clean":"2024-09-13T14:50:10.495247+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:50:10.495247+0000","last_undegraded":"2024-09-13T14:50:10.495247+0000","last_fullsized":"2024-09-13T14:50:10.495247+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"381'4672","reported_seq":4693,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.829207+0000","last_change":"2024-09-13T14:50:00.002592+0000","last_active":"2024-09-13T14:50:50.829207+0000","last_peered":"2024-09-13T14:50:50.829207+0000","last_clean":"2024-09-13T14:50:50.829207+0000","last_became_active":"2024-09-13T14:50:00.002051+0000","last_became_peered":"2024-09-13T14:50:00.002051+0000","last_unstale":"2024-09-13T14:50:50.829207+0000","last_undegraded":"2024-09-13T14:50:50.829207+0000","last_fullsized":"2024-09-13T14:50:50.829207+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4672,"log_dups_size":0,"ondisk_log_size":4672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:28:44.776933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4672,"num_write_kb":299008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":198,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530009+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:50:09.530009+0000","last_peered":"2024-09-13T14:50:09.530009+0000","last_clean":"2024-09-13T14:50:09.530009+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:50:09.530009+0000","last_undegraded":"2024-09-13T14:50:09.530009+0000","last_fullsized":"2024-09-13T14:50:09.530009+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":650,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407473+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:50:50.407473+0000","last_peered":"2024-09-13T14:50:50.407473+0000","last_clean":"2024-09-13T14:50:50.407473+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:50:50.407473+0000","last_undegraded":"2024-09-13T14:50:50.407473+0000","last_fullsized":"2024-09-13T14:50:50.407473+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":610,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745311+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:50:29.745311+0000","last_peered":"2024-09-13T14:50:29.745311+0000","last_clean":"2024-09-13T14:50:29.745311+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:50:29.745311+0000","last_undegraded":"2024-09-13T14:50:29.745311+0000","last_fullsized":"2024-09-13T14:50:29.745311+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"381'4511","reported_seq":4532,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.910597+0000","last_change":"2024-09-13T14:49:59.509238+0000","last_active":"2024-09-13T14:50:50.910597+0000","last_peered":"2024-09-13T14:50:50.910597+0000","last_clean":"2024-09-13T14:50:50.910597+0000","last_became_active":"2024-09-13T14:49:59.508811+0000","last_became_peered":"2024-09-13T14:49:59.508811+0000","last_unstale":"2024-09-13T14:50:50.910597+0000","last_undegraded":"2024-09-13T14:50:50.910597+0000","last_fullsized":"2024-09-13T14:50:50.910597+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4511,"log_dups_size":0,"ondisk_log_size":4511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:43.422288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295632896,"num_objects":4511,"num_object_clones":0,"num_object_copies":13533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4511,"num_write_kb":288704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":482,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495127+0000","last_change":"2024-09-13T14:30:06.421417+0000","last_active":"2024-09-13T14:50:10.495127+0000","last_peered":"2024-09-13T14:50:10.495127+0000","last_clean":"2024-09-13T14:50:10.495127+0000","last_became_active":"2024-09-13T14:30:05.413322+0000","last_became_peered":"2024-09-13T14:30:05.413322+0000","last_unstale":"2024-09-13T14:50:10.495127+0000","last_undegraded":"2024-09-13T14:50:10.495127+0000","last_fullsized":"2024-09-13T14:50:10.495127+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:23.438326+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":904,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529474+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:50:09.529474+0000","last_peered":"2024-09-13T14:50:09.529474+0000","last_clean":"2024-09-13T14:50:09.529474+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:50:09.529474+0000","last_undegraded":"2024-09-13T14:50:09.529474+0000","last_fullsized":"2024-09-13T14:50:09.529474+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":603,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407684+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:50:50.407684+0000","last_peered":"2024-09-13T14:50:50.407684+0000","last_clean":"2024-09-13T14:50:50.407684+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:50:50.407684+0000","last_undegraded":"2024-09-13T14:50:50.407684+0000","last_fullsized":"2024-09-13T14:50:50.407684+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":346,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530400+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:50:09.530400+0000","last_peered":"2024-09-13T14:50:09.530400+0000","last_clean":"2024-09-13T14:50:09.530400+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:50:09.530400+0000","last_undegraded":"2024-09-13T14:50:09.530400+0000","last_fullsized":"2024-09-13T14:50:09.530400+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"380'4269","reported_seq":4288,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:49.145631+0000","last_change":"2024-09-13T14:49:59.507963+0000","last_active":"2024-09-13T14:50:49.145631+0000","last_peered":"2024-09-13T14:50:49.145631+0000","last_clean":"2024-09-13T14:50:49.145631+0000","last_became_active":"2024-09-13T14:49:59.507611+0000","last_became_peered":"2024-09-13T14:49:59.507611+0000","last_unstale":"2024-09-13T14:50:49.145631+0000","last_undegraded":"2024-09-13T14:50:49.145631+0000","last_fullsized":"2024-09-13T14:50:49.145631+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4269,"log_dups_size":0,"ondisk_log_size":4269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:32.773088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279773184,"num_objects":4269,"num_object_clones":0,"num_object_copies":12807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4269,"num_write_kb":273216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":455,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744800+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:50:29.744800+0000","last_peered":"2024-09-13T14:50:29.744800+0000","last_clean":"2024-09-13T14:50:29.744800+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:50:29.744800+0000","last_undegraded":"2024-09-13T14:50:29.744800+0000","last_fullsized":"2024-09-13T14:50:29.744800+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":770,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495035+0000","last_change":"2024-09-13T14:34:46.649374+0000","last_active":"2024-09-13T14:50:10.495035+0000","last_peered":"2024-09-13T14:50:10.495035+0000","last_clean":"2024-09-13T14:50:10.495035+0000","last_became_active":"2024-09-13T14:34:46.649024+0000","last_became_peered":"2024-09-13T14:34:46.649024+0000","last_unstale":"2024-09-13T14:50:10.495035+0000","last_undegraded":"2024-09-13T14:50:10.495035+0000","last_fullsized":"2024-09-13T14:50:10.495035+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:36.031613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":651,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739384+0000","last_change":"2024-09-13T14:40:26.263083+0000","last_active":"2024-09-13T14:50:29.739384+0000","last_peered":"2024-09-13T14:50:29.739384+0000","last_clean":"2024-09-13T14:50:29.739384+0000","last_became_active":"2024-09-13T14:40:26.262669+0000","last_became_peered":"2024-09-13T14:40:26.262669+0000","last_unstale":"2024-09-13T14:50:29.739384+0000","last_undegraded":"2024-09-13T14:50:29.739384+0000","last_fullsized":"2024-09-13T14:50:29.739384+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:36.304355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":655,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739689+0000","last_change":"2024-09-13T14:34:46.578449+0000","last_active":"2024-09-13T14:50:29.739689+0000","last_peered":"2024-09-13T14:50:29.739689+0000","last_clean":"2024-09-13T14:50:29.739689+0000","last_became_active":"2024-09-13T14:34:46.578152+0000","last_became_peered":"2024-09-13T14:34:46.578152+0000","last_unstale":"2024-09-13T14:50:29.739689+0000","last_undegraded":"2024-09-13T14:50:29.739689+0000","last_fullsized":"2024-09-13T14:50:29.739689+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:48.126235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"380'4272","reported_seq":4284,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.976019+0000","last_change":"2024-09-13T14:49:59.509683+0000","last_active":"2024-09-13T14:50:47.976019+0000","last_peered":"2024-09-13T14:50:47.976019+0000","last_clean":"2024-09-13T14:50:47.976019+0000","last_became_active":"2024-09-13T14:49:59.509390+0000","last_became_peered":"2024-09-13T14:49:59.509390+0000","last_unstale":"2024-09-13T14:50:47.976019+0000","last_undegraded":"2024-09-13T14:50:47.976019+0000","last_fullsized":"2024-09-13T14:50:47.976019+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4272,"log_dups_size":0,"ondisk_log_size":4272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:08.503709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279969792,"num_objects":4272,"num_object_clones":0,"num_object_copies":12816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4272,"num_write_kb":273408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":511,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495279+0000","last_change":"2024-09-13T14:40:26.260552+0000","last_active":"2024-09-13T14:50:10.495279+0000","last_peered":"2024-09-13T14:50:10.495279+0000","last_clean":"2024-09-13T14:50:10.495279+0000","last_became_active":"2024-09-13T14:40:26.260101+0000","last_became_peered":"2024-09-13T14:40:26.260101+0000","last_unstale":"2024-09-13T14:50:10.495279+0000","last_undegraded":"2024-09-13T14:50:10.495279+0000","last_fullsized":"2024-09-13T14:50:10.495279+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:54.980606+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":621,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.414038+0000","last_change":"2024-09-13T14:39:23.286889+0000","last_active":"2024-09-13T14:50:50.414038+0000","last_peered":"2024-09-13T14:50:50.414038+0000","last_clean":"2024-09-13T14:50:50.414038+0000","last_became_active":"2024-09-13T14:39:23.285776+0000","last_became_peered":"2024-09-13T14:39:23.285776+0000","last_unstale":"2024-09-13T14:50:50.414038+0000","last_undegraded":"2024-09-13T14:50:50.414038+0000","last_fullsized":"2024-09-13T14:50:50.414038+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:27:07.687996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":588,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744629+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:50:29.744629+0000","last_peered":"2024-09-13T14:50:29.744629+0000","last_clean":"2024-09-13T14:50:29.744629+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:50:29.744629+0000","last_undegraded":"2024-09-13T14:50:29.744629+0000","last_fullsized":"2024-09-13T14:50:29.744629+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530782+0000","last_change":"2024-09-13T14:40:44.707152+0000","last_active":"2024-09-13T14:50:09.530782+0000","last_peered":"2024-09-13T14:50:09.530782+0000","last_clean":"2024-09-13T14:50:09.530782+0000","last_became_active":"2024-09-13T14:40:44.706856+0000","last_became_peered":"2024-09-13T14:40:44.706856+0000","last_unstale":"2024-09-13T14:50:09.530782+0000","last_undegraded":"2024-09-13T14:50:09.530782+0000","last_fullsized":"2024-09-13T14:50:09.530782+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:03.775926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"380'4396","reported_seq":4415,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:49.121486+0000","last_change":"2024-09-13T14:49:59.510827+0000","last_active":"2024-09-13T14:50:49.121486+0000","last_peered":"2024-09-13T14:50:49.121486+0000","last_clean":"2024-09-13T14:50:49.121486+0000","last_became_active":"2024-09-13T14:49:59.509711+0000","last_became_peered":"2024-09-13T14:49:59.509711+0000","last_unstale":"2024-09-13T14:50:49.121486+0000","last_undegraded":"2024-09-13T14:50:49.121486+0000","last_fullsized":"2024-09-13T14:50:49.121486+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4396,"log_dups_size":0,"ondisk_log_size":4396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:17.400887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4396,"num_write_kb":281344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":198,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529690+0000","last_change":"2024-09-13T14:49:47.727307+0000","last_active":"2024-09-13T14:50:09.529690+0000","last_peered":"2024-09-13T14:50:09.529690+0000","last_clean":"2024-09-13T14:50:09.529690+0000","last_became_active":"2024-09-13T14:40:44.708183+0000","last_became_peered":"2024-09-13T14:40:44.708183+0000","last_unstale":"2024-09-13T14:50:09.529690+0000","last_undegraded":"2024-09-13T14:50:09.529690+0000","last_fullsized":"2024-09-13T14:50:09.529690+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:30:33.609834+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":995,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530057+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:50:09.530057+0000","last_peered":"2024-09-13T14:50:09.530057+0000","last_clean":"2024-09-13T14:50:09.530057+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:50:09.530057+0000","last_undegraded":"2024-09-13T14:50:09.530057+0000","last_fullsized":"2024-09-13T14:50:09.530057+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494915+0000","last_change":"2024-09-13T14:40:26.256262+0000","last_active":"2024-09-13T14:50:10.494915+0000","last_peered":"2024-09-13T14:50:10.494915+0000","last_clean":"2024-09-13T14:50:10.494915+0000","last_became_active":"2024-09-13T14:40:26.255927+0000","last_became_peered":"2024-09-13T14:40:26.255927+0000","last_unstale":"2024-09-13T14:50:10.494915+0000","last_undegraded":"2024-09-13T14:50:10.494915+0000","last_fullsized":"2024-09-13T14:50:10.494915+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:07:18.103864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":608,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744137+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:50:29.744137+0000","last_peered":"2024-09-13T14:50:29.744137+0000","last_clean":"2024-09-13T14:50:29.744137+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:50:29.744137+0000","last_undegraded":"2024-09-13T14:50:29.744137+0000","last_fullsized":"2024-09-13T14:50:29.744137+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"380'4363","reported_seq":4375,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.994731+0000","last_change":"2024-09-13T14:49:59.507705+0000","last_active":"2024-09-13T14:50:47.994731+0000","last_peered":"2024-09-13T14:50:47.994731+0000","last_clean":"2024-09-13T14:50:47.994731+0000","last_became_active":"2024-09-13T14:49:59.507374+0000","last_became_peered":"2024-09-13T14:49:59.507374+0000","last_unstale":"2024-09-13T14:50:47.994731+0000","last_undegraded":"2024-09-13T14:50:47.994731+0000","last_fullsized":"2024-09-13T14:50:47.994731+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4363,"log_dups_size":0,"ondisk_log_size":4363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:22.009750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285933568,"num_objects":4363,"num_object_clones":0,"num_object_copies":13089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4363,"num_write_kb":279232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529907+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:50:09.529907+0000","last_peered":"2024-09-13T14:50:09.529907+0000","last_clean":"2024-09-13T14:50:09.529907+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:50:09.529907+0000","last_undegraded":"2024-09-13T14:50:09.529907+0000","last_fullsized":"2024-09-13T14:50:09.529907+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":325,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529780+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:50:09.529780+0000","last_peered":"2024-09-13T14:50:09.529780+0000","last_clean":"2024-09-13T14:50:09.529780+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:50:09.529780+0000","last_undegraded":"2024-09-13T14:50:09.529780+0000","last_fullsized":"2024-09-13T14:50:09.529780+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1187,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744485+0000","last_change":"2024-09-13T14:34:46.571809+0000","last_active":"2024-09-13T14:50:29.744485+0000","last_peered":"2024-09-13T14:50:29.744485+0000","last_clean":"2024-09-13T14:50:29.744485+0000","last_became_active":"2024-09-13T14:34:46.571372+0000","last_became_peered":"2024-09-13T14:34:46.571372+0000","last_unstale":"2024-09-13T14:50:29.744485+0000","last_undegraded":"2024-09-13T14:50:29.744485+0000","last_fullsized":"2024-09-13T14:50:29.744485+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:28.235166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":589,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745746+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:50:29.745746+0000","last_peered":"2024-09-13T14:50:29.745746+0000","last_clean":"2024-09-13T14:50:29.745746+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:50:29.745746+0000","last_undegraded":"2024-09-13T14:50:29.745746+0000","last_fullsized":"2024-09-13T14:50:29.745746+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"380'4347","reported_seq":4359,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.658572+0000","last_change":"2024-09-13T14:49:59.754749+0000","last_active":"2024-09-13T14:50:47.658572+0000","last_peered":"2024-09-13T14:50:47.658572+0000","last_clean":"2024-09-13T14:50:47.658572+0000","last_became_active":"2024-09-13T14:49:59.754398+0000","last_became_peered":"2024-09-13T14:49:59.754398+0000","last_unstale":"2024-09-13T14:50:47.658572+0000","last_undegraded":"2024-09-13T14:50:47.658572+0000","last_fullsized":"2024-09-13T14:50:47.658572+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4347,"log_dups_size":0,"ondisk_log_size":4347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:08.387473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284884992,"num_objects":4347,"num_object_clones":0,"num_object_copies":13041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4347,"num_write_kb":278208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530519+0000","last_change":"2024-09-13T14:30:38.471171+0000","last_active":"2024-09-13T14:50:09.530519+0000","last_peered":"2024-09-13T14:50:09.530519+0000","last_clean":"2024-09-13T14:50:09.530519+0000","last_became_active":"2024-09-13T14:30:11.437310+0000","last_became_peered":"2024-09-13T14:30:11.437310+0000","last_unstale":"2024-09-13T14:50:09.530519+0000","last_undegraded":"2024-09-13T14:50:09.530519+0000","last_fullsized":"2024-09-13T14:50:09.530519+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:40.460840+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":598,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744380+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:50:29.744380+0000","last_peered":"2024-09-13T14:50:29.744380+0000","last_clean":"2024-09-13T14:50:29.744380+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:50:29.744380+0000","last_undegraded":"2024-09-13T14:50:29.744380+0000","last_fullsized":"2024-09-13T14:50:29.744380+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":774,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495956+0000","last_change":"2024-09-13T14:40:29.301549+0000","last_active":"2024-09-13T14:50:10.495956+0000","last_peered":"2024-09-13T14:50:10.495956+0000","last_clean":"2024-09-13T14:50:10.495956+0000","last_became_active":"2024-09-13T14:40:29.301134+0000","last_became_peered":"2024-09-13T14:40:29.301134+0000","last_unstale":"2024-09-13T14:50:10.495956+0000","last_undegraded":"2024-09-13T14:50:10.495956+0000","last_fullsized":"2024-09-13T14:50:10.495956+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:55:04.008821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":634,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739724+0000","last_change":"2024-09-13T14:40:44.707145+0000","last_active":"2024-09-13T14:50:29.739724+0000","last_peered":"2024-09-13T14:50:29.739724+0000","last_clean":"2024-09-13T14:50:29.739724+0000","last_became_active":"2024-09-13T14:40:44.706815+0000","last_became_peered":"2024-09-13T14:40:44.706815+0000","last_unstale":"2024-09-13T14:50:29.739724+0000","last_undegraded":"2024-09-13T14:50:29.739724+0000","last_fullsized":"2024-09-13T14:50:29.739724+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:02.838144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"380'4277","reported_seq":4289,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:47.653360+0000","last_change":"2024-09-13T14:49:59.511444+0000","last_active":"2024-09-13T14:50:47.653360+0000","last_peered":"2024-09-13T14:50:47.653360+0000","last_clean":"2024-09-13T14:50:47.653360+0000","last_became_active":"2024-09-13T14:49:59.511154+0000","last_became_peered":"2024-09-13T14:49:59.511154+0000","last_unstale":"2024-09-13T14:50:47.653360+0000","last_undegraded":"2024-09-13T14:50:47.653360+0000","last_fullsized":"2024-09-13T14:50:47.653360+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4277,"log_dups_size":0,"ondisk_log_size":4277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:18.382270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4277,"num_write_kb":273728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494777+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:50:10.494777+0000","last_peered":"2024-09-13T14:50:10.494777+0000","last_clean":"2024-09-13T14:50:10.494777+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:50:10.494777+0000","last_undegraded":"2024-09-13T14:50:10.494777+0000","last_fullsized":"2024-09-13T14:50:10.494777+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":636,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407321+0000","last_change":"2024-09-13T14:24:00.659370+0000","last_active":"2024-09-13T14:50:50.407321+0000","last_peered":"2024-09-13T14:50:50.407321+0000","last_clean":"2024-09-13T14:50:50.407321+0000","last_became_active":"2024-09-13T14:24:00.658084+0000","last_became_peered":"2024-09-13T14:24:00.658084+0000","last_unstale":"2024-09-13T14:50:50.407321+0000","last_undegraded":"2024-09-13T14:50:50.407321+0000","last_fullsized":"2024-09-13T14:50:50.407321+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:36:58.848846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":974,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.747414+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:50:29.747414+0000","last_peered":"2024-09-13T14:50:29.747414+0000","last_clean":"2024-09-13T14:50:29.747414+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:50:29.747414+0000","last_undegraded":"2024-09-13T14:50:29.747414+0000","last_fullsized":"2024-09-13T14:50:29.747414+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":634,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407089+0000","last_change":"2024-09-13T14:24:00.652462+0000","last_active":"2024-09-13T14:50:50.407089+0000","last_peered":"2024-09-13T14:50:50.407089+0000","last_clean":"2024-09-13T14:50:50.407089+0000","last_became_active":"2024-09-13T14:24:00.651818+0000","last_became_peered":"2024-09-13T14:24:00.651818+0000","last_unstale":"2024-09-13T14:50:50.407089+0000","last_undegraded":"2024-09-13T14:50:50.407089+0000","last_fullsized":"2024-09-13T14:50:50.407089+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:56.301068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"381'4514","reported_seq":4535,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.922894+0000","last_change":"2024-09-13T14:49:59.755421+0000","last_active":"2024-09-13T14:50:50.922894+0000","last_peered":"2024-09-13T14:50:50.922894+0000","last_clean":"2024-09-13T14:50:50.922894+0000","last_became_active":"2024-09-13T14:49:59.755094+0000","last_became_peered":"2024-09-13T14:49:59.755094+0000","last_unstale":"2024-09-13T14:50:50.922894+0000","last_undegraded":"2024-09-13T14:50:50.922894+0000","last_fullsized":"2024-09-13T14:50:50.922894+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":4514,"log_dups_size":0,"ondisk_log_size":4514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:27.511205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295829504,"num_objects":4514,"num_object_clones":0,"num_object_copies":13542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4514,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":487,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.410892+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:50:50.410892+0000","last_peered":"2024-09-13T14:50:50.410892+0000","last_clean":"2024-09-13T14:50:50.410892+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:50:50.410892+0000","last_undegraded":"2024-09-13T14:50:50.410892+0000","last_fullsized":"2024-09-13T14:50:50.410892+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":669,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739407+0000","last_change":"2024-09-13T14:40:26.272055+0000","last_active":"2024-09-13T14:50:29.739407+0000","last_peered":"2024-09-13T14:50:29.739407+0000","last_clean":"2024-09-13T14:50:29.739407+0000","last_became_active":"2024-09-13T14:40:26.265740+0000","last_became_peered":"2024-09-13T14:40:26.265740+0000","last_unstale":"2024-09-13T14:50:29.739407+0000","last_undegraded":"2024-09-13T14:50:29.739407+0000","last_fullsized":"2024-09-13T14:50:29.739407+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:35.727104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":798,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530072+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:50:09.530072+0000","last_peered":"2024-09-13T14:50:09.530072+0000","last_clean":"2024-09-13T14:50:09.530072+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:50:09.530072+0000","last_undegraded":"2024-09-13T14:50:09.530072+0000","last_fullsized":"2024-09-13T14:50:09.530072+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495721+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:50:10.495721+0000","last_peered":"2024-09-13T14:50:10.495721+0000","last_clean":"2024-09-13T14:50:10.495721+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:50:10.495721+0000","last_undegraded":"2024-09-13T14:50:10.495721+0000","last_fullsized":"2024-09-13T14:50:10.495721+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494751+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:50:10.494751+0000","last_peered":"2024-09-13T14:50:10.494751+0000","last_clean":"2024-09-13T14:50:10.494751+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:50:10.494751+0000","last_undegraded":"2024-09-13T14:50:10.494751+0000","last_fullsized":"2024-09-13T14:50:10.494751+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":89,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494868+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:50:10.494868+0000","last_peered":"2024-09-13T14:50:10.494868+0000","last_clean":"2024-09-13T14:50:10.494868+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:50:10.494868+0000","last_undegraded":"2024-09-13T14:50:10.494868+0000","last_fullsized":"2024-09-13T14:50:10.494868+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":585,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.745622+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:50:29.745622+0000","last_peered":"2024-09-13T14:50:29.745622+0000","last_clean":"2024-09-13T14:50:29.745622+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:50:29.745622+0000","last_undegraded":"2024-09-13T14:50:29.745622+0000","last_fullsized":"2024-09-13T14:50:29.745622+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":379,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495727+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:50:10.495727+0000","last_peered":"2024-09-13T14:50:10.495727+0000","last_clean":"2024-09-13T14:50:10.495727+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:50:10.495727+0000","last_undegraded":"2024-09-13T14:50:10.495727+0000","last_fullsized":"2024-09-13T14:50:10.495727+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530774+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:50:09.530774+0000","last_peered":"2024-09-13T14:50:09.530774+0000","last_clean":"2024-09-13T14:50:09.530774+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:50:09.530774+0000","last_undegraded":"2024-09-13T14:50:09.530774+0000","last_fullsized":"2024-09-13T14:50:09.530774+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":592,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.413789+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:50:50.413789+0000","last_peered":"2024-09-13T14:50:50.413789+0000","last_clean":"2024-09-13T14:50:50.413789+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:50:50.413789+0000","last_undegraded":"2024-09-13T14:50:50.413789+0000","last_fullsized":"2024-09-13T14:50:50.413789+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":642,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.741190+0000","last_change":"2024-09-13T14:40:44.708411+0000","last_active":"2024-09-13T14:50:29.741190+0000","last_peered":"2024-09-13T14:50:29.741190+0000","last_clean":"2024-09-13T14:50:29.741190+0000","last_became_active":"2024-09-13T14:40:44.707919+0000","last_became_peered":"2024-09-13T14:40:44.707919+0000","last_unstale":"2024-09-13T14:50:29.741190+0000","last_undegraded":"2024-09-13T14:50:29.741190+0000","last_fullsized":"2024-09-13T14:50:29.741190+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:31:19.077784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":648,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407001+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:50:50.407001+0000","last_peered":"2024-09-13T14:50:50.407001+0000","last_clean":"2024-09-13T14:50:50.407001+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:50:50.407001+0000","last_undegraded":"2024-09-13T14:50:50.407001+0000","last_fullsized":"2024-09-13T14:50:50.407001+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529991+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:50:09.529991+0000","last_peered":"2024-09-13T14:50:09.529991+0000","last_clean":"2024-09-13T14:50:09.529991+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:50:09.529991+0000","last_undegraded":"2024-09-13T14:50:09.529991+0000","last_fullsized":"2024-09-13T14:50:09.529991+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":666,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407253+0000","last_change":"2024-09-13T14:34:46.565099+0000","last_active":"2024-09-13T14:50:50.407253+0000","last_peered":"2024-09-13T14:50:50.407253+0000","last_clean":"2024-09-13T14:50:50.407253+0000","last_became_active":"2024-09-13T14:34:46.563874+0000","last_became_peered":"2024-09-13T14:34:46.563874+0000","last_unstale":"2024-09-13T14:50:50.407253+0000","last_undegraded":"2024-09-13T14:50:50.407253+0000","last_fullsized":"2024-09-13T14:50:50.407253+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:31.384455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":634,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739606+0000","last_change":"2024-09-13T14:24:00.657127+0000","last_active":"2024-09-13T14:50:29.739606+0000","last_peered":"2024-09-13T14:50:29.739606+0000","last_clean":"2024-09-13T14:50:29.739606+0000","last_became_active":"2024-09-13T14:24:00.656852+0000","last_became_peered":"2024-09-13T14:24:00.656852+0000","last_unstale":"2024-09-13T14:50:29.739606+0000","last_undegraded":"2024-09-13T14:50:29.739606+0000","last_fullsized":"2024-09-13T14:50:29.739606+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:20.534695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":610,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744066+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:50:29.744066+0000","last_peered":"2024-09-13T14:50:29.744066+0000","last_clean":"2024-09-13T14:50:29.744066+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:50:29.744066+0000","last_undegraded":"2024-09-13T14:50:29.744066+0000","last_fullsized":"2024-09-13T14:50:29.744066+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":661,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.406606+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:50:50.406606+0000","last_peered":"2024-09-13T14:50:50.406606+0000","last_clean":"2024-09-13T14:50:50.406606+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:50:50.406606+0000","last_undegraded":"2024-09-13T14:50:50.406606+0000","last_fullsized":"2024-09-13T14:50:50.406606+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":78,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495129+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:50:10.495129+0000","last_peered":"2024-09-13T14:50:10.495129+0000","last_clean":"2024-09-13T14:50:10.495129+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:50:10.495129+0000","last_undegraded":"2024-09-13T14:50:10.495129+0000","last_fullsized":"2024-09-13T14:50:10.495129+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":556,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.744649+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:50:29.744649+0000","last_peered":"2024-09-13T14:50:29.744649+0000","last_clean":"2024-09-13T14:50:29.744649+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:50:29.744649+0000","last_undegraded":"2024-09-13T14:50:29.744649+0000","last_fullsized":"2024-09-13T14:50:29.744649+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":637,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.739587+0000","last_change":"2024-09-13T14:40:26.263098+0000","last_active":"2024-09-13T14:50:29.739587+0000","last_peered":"2024-09-13T14:50:29.739587+0000","last_clean":"2024-09-13T14:50:29.739587+0000","last_became_active":"2024-09-13T14:40:26.262687+0000","last_became_peered":"2024-09-13T14:40:26.262687+0000","last_unstale":"2024-09-13T14:50:29.739587+0000","last_undegraded":"2024-09-13T14:50:29.739587+0000","last_fullsized":"2024-09-13T14:50:29.739587+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:26.482962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495021+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:50:10.495021+0000","last_peered":"2024-09-13T14:50:10.495021+0000","last_clean":"2024-09-13T14:50:10.495021+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:50:10.495021+0000","last_undegraded":"2024-09-13T14:50:10.495021+0000","last_fullsized":"2024-09-13T14:50:10.495021+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":345,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.529572+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:50:09.529572+0000","last_peered":"2024-09-13T14:50:09.529572+0000","last_clean":"2024-09-13T14:50:09.529572+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:50:09.529572+0000","last_undegraded":"2024-09-13T14:50:09.529572+0000","last_fullsized":"2024-09-13T14:50:09.529572+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.495766+0000","last_change":"2024-09-13T14:40:26.259645+0000","last_active":"2024-09-13T14:50:10.495766+0000","last_peered":"2024-09-13T14:50:10.495766+0000","last_clean":"2024-09-13T14:50:10.495766+0000","last_became_active":"2024-09-13T14:40:26.259335+0000","last_became_peered":"2024-09-13T14:40:26.259335+0000","last_unstale":"2024-09-13T14:50:10.495766+0000","last_undegraded":"2024-09-13T14:50:10.495766+0000","last_fullsized":"2024-09-13T14:50:10.495766+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:42.906804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":311,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:09.530064+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:50:09.530064+0000","last_peered":"2024-09-13T14:50:09.530064+0000","last_clean":"2024-09-13T14:50:09.530064+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:50:09.530064+0000","last_undegraded":"2024-09-13T14:50:09.530064+0000","last_fullsized":"2024-09-13T14:50:09.530064+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":658,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.414122+0000","last_change":"2024-09-13T14:24:00.659466+0000","last_active":"2024-09-13T14:50:50.414122+0000","last_peered":"2024-09-13T14:50:50.414122+0000","last_clean":"2024-09-13T14:50:50.414122+0000","last_became_active":"2024-09-13T14:24:00.658150+0000","last_became_peered":"2024-09-13T14:24:00.658150+0000","last_unstale":"2024-09-13T14:50:50.414122+0000","last_undegraded":"2024-09-13T14:50:50.414122+0000","last_fullsized":"2024-09-13T14:50:50.414122+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:48.252538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":645,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-09-13T14:50:50.407073+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:50:50.407073+0000","last_peered":"2024-09-13T14:50:50.407073+0000","last_clean":"2024-09-13T14:50:50.407073+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:50:50.407073+0000","last_undegraded":"2024-09-13T14:50:50.407073+0000","last_fullsized":"2024-09-13T14:50:50.407073+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-09-13T14:50:10.494619+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:50:10.494619+0000","last_peered":"2024-09-13T14:50:10.494619+0000","last_clean":"2024-09-13T14:50:10.494619+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:50:10.494619+0000","last_undegraded":"2024-09-13T14:50:10.494619+0000","last_fullsized":"2024-09-13T14:50:10.494619+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":648,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-09-13T14:50:29.741102+0000","last_change":"2024-09-13T14:40:44.708775+0000","last_active":"2024-09-13T14:50:29.741102+0000","last_peered":"2024-09-13T14:50:29.741102+0000","last_clean":"2024-09-13T14:50:29.741102+0000","last_became_active":"2024-09-13T14:40:44.708057+0000","last_became_peered":"2024-09-13T14:40:44.708057+0000","last_unstale":"2024-09-13T14:50:29.741102+0000","last_undegraded":"2024-09-13T14:50:29.741102+0000","last_fullsized":"2024-09-13T14:50:29.741102+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.200733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4617732096,"num_objects":70461,"num_object_clones":0,"num_object_copies":211383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":70461,"num_write_kb":4509504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13763018752,"data_stored":13763018752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70461,"ondisk_log_size":70461,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397976,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3366176,"kb_used_data":2470168,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":90378976,"statfs":{"total":95995035648,"available":92548071424,"internally_reserved":0,"allocated":2529452032,"data_stored":2527405199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594204,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931792,"kb_used_data":1041480,"kb_used_omap":1,"kb_used_meta":890302,"kb_avail":91813360,"statfs":{"total":95995035648,"available":94016880640,"internally_reserved":0,"allocated":1066475520,"data_stored":1064444369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911669592},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665658,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2797900,"kb_used_data":2461188,"kb_used_omap":9,"kb_used_meta":336694,"kb_avail":90947252,"statfs":{"total":95995035648,"available":93129986048,"internally_reserved":0,"allocated":2520256512,"data_stored":2518225764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":344774856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986660,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1728860,"kb_used_data":833172,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92016292,"statfs":{"total":95995035648,"available":94224683008,"internally_reserved":0,"allocated":853168128,"data_stored":851125321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697220,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3432732,"kb_used_data":2572240,"kb_used_omap":4,"kb_used_meta":860475,"kb_avail":90312420,"statfs":{"total":95995035648,"available":92479918080,"internally_reserved":0,"allocated":2633973760,"data_stored":2631939139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":881126475},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860746,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1483984,"kb_used_data":808712,"kb_used_omap":8,"kb_used_meta":675255,"kb_avail":92261168,"statfs":{"total":95995035648,"available":94475436032,"internally_reserved":0,"allocated":828121088,"data_stored":826086839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":691461964},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020418,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2981128,"kb_used_data":2591488,"kb_used_omap":9,"kb_used_meta":389622,"kb_avail":90764024,"statfs":{"total":95995035648,"available":92942360576,"internally_reserved":0,"allocated":2653683712,"data_stored":2651656518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":398973765},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187808,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1778312,"kb_used_data":1115712,"kb_used_omap":7,"kb_used_meta":662584,"kb_avail":91966840,"statfs":{"total":95995035648,"available":94174044160,"internally_reserved":0,"allocated":1142489088,"data_stored":1140462160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":678486347},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1142292480,"data_stored":1142292480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2668429312,"data_stored":2668429312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":835584000,"data_stored":835584000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2397110272,"data_stored":2397110272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":850460672,"data_stored":850460672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2522480640,"data_stored":2522480640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":826540032,"data_stored":826540032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2520121344,"data_stored":2520121344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:50:54.203 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:50:54.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_4 pg_num 26 2024-09-13T14:50:54.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:54.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:54.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:54.397 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:54.424 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:54.431 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:54 smithi189 ceph-mon[90992]: from='client.56194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:54.448 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:54.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:54.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:54 smithi017 ceph-mon[103001]: from='client.56194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:54 smithi017 ceph-mon[103001]: pgmap v1283: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:50:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:54 smithi017 ceph-mon[107556]: from='client.56194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:54 smithi017 ceph-mon[107556]: pgmap v1283: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:50:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.703 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:54.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:54 smithi189 ceph-mon[90992]: pgmap v1283: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:50:54.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:54.810 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 77950 77934 115.937 76.375 0.00755388 0.0086198 2024-09-13T14:50:54.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:54.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:54.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:55.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:55.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:55.364 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.398 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:55.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:55.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:55.624 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:55.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:55.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:55.810 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 79144 79128 114.976 74.625 0.0170665 0.00869215 2024-09-13T14:50:55.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:56.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:56.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.077 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:56.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.174 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:56.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:56.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:56.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:56.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:50:56.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:56.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:56.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.643 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:56.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:56 smithi017 ceph-mon[107556]: pgmap v1284: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:50:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:56 smithi017 ceph-mon[103001]: pgmap v1284: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:50:56.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:56 smithi189 ceph-mon[90992]: pgmap v1284: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:50:56.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:56.811 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 80204 80188 113.868 66.25 0.0160671 0.00877728 2024-09-13T14:50:56.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:56.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:57.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:57.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:57.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.229 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:57.367 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:57.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:50:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:50:57] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-09-13T14:50:57.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:57.490 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:57.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:57.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:57.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:57.811 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 81336 81320 112.91 70.75 0.0142225 0.00885153 2024-09-13T14:50:57.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:57.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:58.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:58.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:58.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:58.144 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:58.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:58.280 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:58.314 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:58.416 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:58.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:58.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[103001]: pgmap v1285: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:50:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:50:58 smithi017 ceph-mon[107556]: pgmap v1285: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:50:58.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:50:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:58.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:50:58 smithi189 ceph-mon[90992]: pgmap v1285: 99 pgs: 99 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-13T14:50:58.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:50:58.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:58.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:58.811 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 82647 82631 112.236 81.9375 0.0112166 0.00889989 2024-09-13T14:50:58.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:58.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:58.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:59.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:59.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:59.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:59.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.811 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 83795 83779 111.375 71.75 0.0143593 0.00897311 2024-09-13T14:50:59.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:50:59.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:00.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:00.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:00.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:00.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:00.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:00.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:00.709 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:00.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:00.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:00.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 85060 85044 110.701 79.0625 0.00758631 0.0090282 2024-09-13T14:51:00.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:00.915 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:51:00.915 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:00.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:00.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:51:00.957 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:00.991 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:51:00.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:00.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:00 smithi017 ceph-mon[103001]: pgmap v1286: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:51:00.992 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:00 smithi017 ceph-mon[107556]: pgmap v1286: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:51:00.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:00.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:01.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:00 smithi189 ceph-mon[90992]: pgmap v1286: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:51:01.027 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:51:01.027 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:01.058 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:51:01.058 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:01.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:01.098 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:51:01.098 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:01.136 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:51:01.137 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:01.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:01.179 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:51:01.179 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:01.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:01.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:01.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:01.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:01.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:01.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:01.652 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:01.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:01.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:01.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 86233 86217 109.938 73.3125 0.00991172 0.00909087 2024-09-13T14:51:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:01.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1737369182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1737369182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:01.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1737369182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:02.069 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.201 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:02.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:02.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:02.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:02.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:02.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:02.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:02.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:02.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:02.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 87393 87377 109.189 72.5 0.00468796 0.00915348 2024-09-13T14:51:02.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:02 smithi189 ceph-mon[90992]: pgmap v1287: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:51:03.037 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:02 smithi017 ceph-mon[107556]: pgmap v1287: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:51:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:02 smithi017 ceph-mon[103001]: pgmap v1287: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:51:03.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:03.318 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.352 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 88293 88277 108.151 56.25 0.0202329 0.00924124 2024-09-13T14:51:03.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:03.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:03.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:04.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:04.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:04.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.295 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:04.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.431 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.564 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.698 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 89501 89485 107.523 75.5 0.00653009 0.0092954 2024-09-13T14:51:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:04.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:04.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:04.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:04 smithi189 ceph-mon[90992]: pgmap v1288: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:51:05.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:05.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:04 smithi017 ceph-mon[103001]: pgmap v1288: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:51:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:04 smithi017 ceph-mon[107556]: pgmap v1288: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:51:05.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:05.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:05.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:05.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.525 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:05.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:05.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:05.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:05.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:05.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 90713 90697 106.924 75.75 0.0126504 0.00934665 2024-09-13T14:51:05.908 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:05.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:06.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:06.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:06.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:06.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:06.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:06.583 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.612 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:06.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 91801 91785 106.203 68 0.0100179 0.00941128 2024-09-13T14:51:06.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:06.954 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:51:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:51:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:51:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:51:06.954 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:51:06.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:06 smithi189 ceph-mon[90992]: pgmap v1289: 99 pgs: 99 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:51:07.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:07.164 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:06 smithi017 ceph-mon[103001]: pgmap v1289: 99 pgs: 99 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:51:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:06 smithi017 ceph-mon[107556]: pgmap v1289: 99 pgs: 99 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:51:07.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:07.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:07.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:07.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:07] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-09-13T14:51:07.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 92910 92894 105.532 69.3125 0.0239371 0.00947045 2024-09-13T14:51:07.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:07.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:07.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.989 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 9 pg_num to 26 2024-09-13T14:51:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1923460244' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:07 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:08.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:08.141 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1923460244' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:07 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1923460244' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:07 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:51:08.242 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.274 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:08.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.420 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:08.556 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:08.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:08.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:08.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:51:08.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.789 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:08.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 94085 94069 104.959 73.4375 0.0110981 0.00952264 2024-09-13T14:51:08.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[103001]: pgmap v1290: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[103001]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[107556]: pgmap v1290: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:51:08.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:08 smithi017 ceph-mon[107556]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:51:08.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:08.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:09.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:09.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.234 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:08 smithi189 ceph-mon[90992]: pgmap v1290: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:51:09.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:51:09.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:08 smithi189 ceph-mon[90992]: osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:51:09.268 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:09.543 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:09.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:09.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:09.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.812 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 95194 95178 104.334 69.3125 0.0133437 0.00957702 2024-09-13T14:51:09.815 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:09.917 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:09.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:09 smithi017 ceph-mon[103001]: pgmap v1292: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:51:09.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:51:09.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:09 smithi017 ceph-mon[107556]: pgmap v1292: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:51:09.997 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:51:10.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:10.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:09 smithi189 ceph-mon[90992]: pgmap v1292: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:51:10.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:51:10.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:10.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:10.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:10.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:10.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:10.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:10.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.813 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 95202 95186 102.544 0.5 1.01409 0.00960316 2024-09-13T14:51:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:10.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:10.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.980 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:11.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:11.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:51:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[103001]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:51:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[107556]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:51:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-mon[90992]: osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:51:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:11.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:11.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.392 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:11.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:11.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.664 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:11.813 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 96404 96388 102.079 75.125 0.00498642 0.00978493 2024-09-13T14:51:11.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:11.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:12.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:12.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:12.177 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[107556]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:51:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[107556]: 9.5 scrub starts 2024-09-13T14:51:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[107556]: pgmap v1295: 109 pgs: 2 peering, 7 unknown, 100 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[107556]: 9.3 deep-scrub starts 2024-09-13T14:51:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[103001]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:51:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[103001]: 9.5 scrub starts 2024-09-13T14:51:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[103001]: pgmap v1295: 109 pgs: 2 peering, 7 unknown, 100 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:11 smithi017 ceph-mon[103001]: 9.3 deep-scrub starts 2024-09-13T14:51:12.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:11 smithi189 ceph-mon[90992]: osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:51:12.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:11 smithi189 ceph-mon[90992]: 9.5 scrub starts 2024-09-13T14:51:12.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:11 smithi189 ceph-mon[90992]: pgmap v1295: 109 pgs: 2 peering, 7 unknown, 100 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:12.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:11 smithi189 ceph-mon[90992]: 9.3 deep-scrub starts 2024-09-13T14:51:12.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:12.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:12.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:12.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:12.592 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:12.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:12.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:12.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.743 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:12.815 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:51:12.814796+0000 min lat: 0.00120371 max lat: 1.06657 avg lat: 0.00986133 2024-09-13T14:51:12.815 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:51:12.815 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 97325 97309 101.334 57.5625 0.0258765 0.00986133 2024-09-13T14:51:12.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.863 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:12.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:12.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:13.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:13.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[107556]: 9.0 scrub starts 2024-09-13T14:51:13.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[107556]: 9.2 scrub starts 2024-09-13T14:51:13.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[107556]: 9.9 scrub starts 2024-09-13T14:51:13.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[107556]: 9.5 scrub ok 2024-09-13T14:51:13.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:13.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[103001]: 9.0 scrub starts 2024-09-13T14:51:13.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[103001]: 9.2 scrub starts 2024-09-13T14:51:13.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[103001]: 9.9 scrub starts 2024-09-13T14:51:13.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[103001]: 9.5 scrub ok 2024-09-13T14:51:13.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:13.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:12 smithi189 ceph-mon[90992]: 9.0 scrub starts 2024-09-13T14:51:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:12 smithi189 ceph-mon[90992]: 9.2 scrub starts 2024-09-13T14:51:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:12 smithi189 ceph-mon[90992]: 9.9 scrub starts 2024-09-13T14:51:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:12 smithi189 ceph-mon[90992]: 9.5 scrub ok 2024-09-13T14:51:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:13.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:13.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:13.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:13.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:13.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:13.619 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:13.721 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.748 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.815 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 98197 98181 100.566 54.5 0.0200727 0.0099382 2024-09-13T14:51:13.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:13.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:13.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.0 scrub ok 2024-09-13T14:51:13.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.4 scrub starts 2024-09-13T14:51:13.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.13 deep-scrub starts 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.9 scrub ok 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.1 scrub starts 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: 9.19 scrub starts 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[103001]: pgmap v1296: 109 pgs: 2 peering, 1 active+clean+scrubbing+deep, 4 unknown, 102 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.0 scrub ok 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.4 scrub starts 2024-09-13T14:51:13.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.13 deep-scrub starts 2024-09-13T14:51:13.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.9 scrub ok 2024-09-13T14:51:13.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.1 scrub starts 2024-09-13T14:51:13.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: 9.19 scrub starts 2024-09-13T14:51:13.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:13 smithi017 ceph-mon[107556]: pgmap v1296: 109 pgs: 2 peering, 1 active+clean+scrubbing+deep, 4 unknown, 102 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:51:13.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.0 scrub ok 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.4 scrub starts 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.13 deep-scrub starts 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.9 scrub ok 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.1 scrub starts 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: 9.19 scrub starts 2024-09-13T14:51:14.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:13 smithi189 ceph-mon[90992]: pgmap v1296: 109 pgs: 2 peering, 1 active+clean+scrubbing+deep, 4 unknown, 102 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:51:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:14.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:14.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:14.290 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:14.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:14.433 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:14.457 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:51:14.458 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:51:14.458 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T14:51:14.458 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:51:14.458 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:51:14.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T14:51:14.477 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:14.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:14.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:14.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 98916 98900 99.6648 44.9375 0.0513424 0.0100274 2024-09-13T14:51:14.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.2 scrub ok 2024-09-13T14:51:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.10 scrub starts 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.1 scrub ok 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.7 deep-scrub starts 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.8 scrub starts 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.4 scrub ok 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.19 scrub ok 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[107556]: 9.15 scrub starts 2024-09-13T14:51:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.2 scrub ok 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.10 scrub starts 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.1 scrub ok 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.7 deep-scrub starts 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.8 scrub starts 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.4 scrub ok 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.19 scrub ok 2024-09-13T14:51:14.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:14 smithi017 ceph-mon[103001]: 9.15 scrub starts 2024-09-13T14:51:14.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.2 scrub ok 2024-09-13T14:51:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.10 scrub starts 2024-09-13T14:51:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.1 scrub ok 2024-09-13T14:51:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.7 deep-scrub starts 2024-09-13T14:51:15.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.8 scrub starts 2024-09-13T14:51:15.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.4 scrub ok 2024-09-13T14:51:15.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.19 scrub ok 2024-09-13T14:51:15.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:14 smithi189 ceph-mon[90992]: 9.15 scrub starts 2024-09-13T14:51:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:15.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:15.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.309 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:15.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.441 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:15.543 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:15.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:15.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:15.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:15.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 99714 99698 98.8746 49.875 0.0102939 0.010108 2024-09-13T14:51:15.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.849 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:51:15.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:15.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:15.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: 9.10 scrub ok 2024-09-13T14:51:16.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: 9.14 scrub starts 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: 9.8 scrub ok 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: 9.11 scrub starts 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: 9.15 scrub ok 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/733575475' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: pgmap v1297: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail 2024-09-13T14:51:16.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:15 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: 9.10 scrub ok 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: 9.14 scrub starts 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: 9.8 scrub ok 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: 9.11 scrub starts 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: 9.15 scrub ok 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/733575475' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: pgmap v1297: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail 2024-09-13T14:51:16.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: 9.10 scrub ok 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: 9.14 scrub starts 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: 9.8 scrub ok 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: 9.11 scrub starts 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: 9.15 scrub ok 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/733575475' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: pgmap v1297: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail 2024-09-13T14:51:16.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:15 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:51:16.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:16.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:16.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:16.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:16.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:16.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:16.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 100040 100024 97.6483 20.375 0.00286658 0.0101141 2024-09-13T14:51:16.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:17.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:17.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:17.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: 9.14 scrub ok 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: 9.14 scrub ok 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:17.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:51:17.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:51:17.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:16 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: 9.14 scrub ok 2024-09-13T14:51:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:51:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:51:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:16 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:17.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:17.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:17.439 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:17] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-09-13T14:51:17.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:17.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:17.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:17.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:17.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:17.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 100040 100024 96.1463 0 - 0.0101141 2024-09-13T14:51:17.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:17.939 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[103001]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:51:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[103001]: 9.12 scrub starts 2024-09-13T14:51:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[103001]: pgmap v1300: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:51:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[107556]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:51:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[107556]: 9.12 scrub starts 2024-09-13T14:51:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[107556]: pgmap v1300: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:51:17.977 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:51:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:51:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:51:17.977 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88010 9.c 9:36a45f6d:::benchmark_data_smithi017_339564_object88009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.595099122000001, 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23807841599999999, 2024-09-13T14:51:17.978 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88010 2024-09-13T14:51:17.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:17.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377063+0000", 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8300000000000001e-06 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377118+0000", 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4843999999999999e-05 2024-09-13T14:51:17.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380264+0000", 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003145994 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380275+0000", 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1649e-05 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:17.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:51:17.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054572799999999999 2024-09-13T14:51:17.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381007+0000", 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018597899999999999 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381155+0000", 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014763800000000001 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387265+0000", 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061102819999999999 2024-09-13T14:51:17.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614900+0000", 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227635122 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615054+0000", 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015373700000000001 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615094+0000", 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0738999999999997e-05 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:17.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615139+0000", 2024-09-13T14:51:17.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4873999999999998e-05 2024-09-13T14:51:17.991 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:17.991 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:17.991 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:17.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88013 9.c 9:3b76033e:::benchmark_data_smithi017_339564_object88012:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.591339116, 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23455462899999999, 2024-09-13T14:51:17.992 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88013 2024-09-13T14:51:17.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:17.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380822+0000", 2024-09-13T14:51:17.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3289999999999999e-06 2024-09-13T14:51:17.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380883+0000", 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0396000000000002e-05 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:17.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380895+0000", 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2823000000000001e-05 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380903+0000", 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8860000000000004e-06 2024-09-13T14:51:17.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384158+0000", 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003254346 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:17.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384318+0000", 2024-09-13T14:51:18.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016008400000000001 2024-09-13T14:51:18.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384451+0000", 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013359999999999999 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614962+0000", 2024-09-13T14:51:18.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.230510786 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615189+0000", 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022717899999999999 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615283+0000", 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3936000000000003e-05 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615340+0000", 2024-09-13T14:51:18.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6261999999999998e-05 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615376+0000", 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6001999999999999e-05 2024-09-13T14:51:18.004 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88014 9.c 9:30066bc1:::benchmark_data_smithi017_339564_object88013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.591243237, 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23458493899999999, 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.005 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88014 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:51:18.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380969+0000", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1617999999999999e-05 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380978+0000", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4979999999999995e-06 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380985+0000", 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:51:18.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387301+0000", 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063155529999999998 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387451+0000", 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150414 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387614+0000", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016278399999999999 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615015+0000", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227400877 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615195+0000", 2024-09-13T14:51:18.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017990500000000001 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615419+0000", 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224207 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615466+0000", 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7579e-05 2024-09-13T14:51:18.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615502+0000", 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5425e-05 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:88018 9.4 e381/376 9:2920a091:::benchmark_data_smithi017_339564_object88017:head v 381'5555, mlcod=381'5553)", 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.589433313000001, 2024-09-13T14:51:18.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23245458799999999, 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382728+0000", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6099999999999999e-07 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382749+0000", 2024-09-13T14:51:18.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1486000000000001e-05 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382760+0000", 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1107999999999999e-05 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382767+0000", 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8820000000000003e-06 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384135+0000", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013674410000000001 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384156+0000", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1027999999999999e-05 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615151+0000", 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23099550899999999 2024-09-13T14:51:18.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615181+0000", 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0173000000000001e-05 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88020 9.c 9:32f382c7:::benchmark_data_smithi017_339564_object88019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.58920764, 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23295475700000001, 2024-09-13T14:51:18.017 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88020 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382954+0000", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1620000000000001e-06 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383001+0000", 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7929000000000001e-05 2024-09-13T14:51:18.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383013+0000", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1324e-05 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383020+0000", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5340000000000002e-06 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390514+0000", 2024-09-13T14:51:18.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074939789999999996 2024-09-13T14:51:18.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390651+0000", 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013669299999999999 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390787+0000", 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135773 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615200+0000", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22441285599999999 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615234+0000", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4634000000000001e-05 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615794+0000", 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055964299999999997 2024-09-13T14:51:18.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615859+0000", 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5318e-05 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615907+0000", 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7911999999999998e-05 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88023 9.2 9:452883ed:::benchmark_data_smithi017_339564_object88022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.583620685, 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.226308658, 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88023 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.025 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388541+0000", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4589999999999999e-06 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388592+0000", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1138999999999999e-05 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388604+0000", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2330999999999999e-05 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388612+0000", 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2749999999999998e-06 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388666+0000", 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4552000000000002e-05 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388799+0000", 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013261499999999999 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388937+0000", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137942 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.392307+0000", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033704109999999998 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393094+0000", 2024-09-13T14:51:18.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078649099999999995 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393918+0000", 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082484500000000005 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393974+0000", 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5798999999999999e-05 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614848+0000", 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22087379900000001 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88025 9.8 9:1a0cff2e:::benchmark_data_smithi017_339564_object88024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.581455397999999, 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23531686499999999, 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.031 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88025 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390706+0000", 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3009999999999999e-06 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390728+0000", 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1959000000000001e-05 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390741+0000", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2892e-05 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390748+0000", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390808+0000", 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0133000000000002e-05 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390954+0000", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014599599999999999 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391077+0000", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012244 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621755+0000", 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23067821399999999 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621803+0000", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8112999999999998e-05 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625781+0000", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039782510000000004 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625827+0000", 2024-09-13T14:51:18.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5982000000000001e-05 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626021+0000", 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019408899999999999 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:88013 9.c e381/376 ondisk, result = 0)", 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:51:18.037 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.581251222000001, 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22407027400000001, 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390910+0000", 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0500000000000002e-07 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390913+0000", 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.76e-06 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390921+0000", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0080000000000002e-06 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390928+0000", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1419999999999998e-06 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614947+0000", 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.224019412 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614955+0000", 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0760000000000003e-06 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614979+0000", 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3971000000000001e-05 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.041 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:88014 9.c e381/376 ondisk, result = 0)", 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.581005189999999, 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22386584100000001, 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391156+0000", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5600000000000005e-07 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391159+0000", 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.49e-06 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391167+0000", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3850000000000002e-06 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391173+0000", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0000000000000002e-06 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615005+0000", 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.223831266 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615011+0000", 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0759999999999999e-06 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615021+0000", 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8679999999999994e-06 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88029 9.8 9:1879118d:::benchmark_data_smithi017_339564_object88028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.579144470999999, 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.233198557, 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88029 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393017+0000", 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.37e-06 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393040+0000", 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2705e-05 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393052+0000", 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1895e-05 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393061+0000", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614926+0000", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.221865756 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615166+0000", 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000239127 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615958+0000", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079216400000000004 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621846+0000", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058882079999999998 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625766+0000", 2024-09-13T14:51:18.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039203490000000001 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626034+0000", 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026819300000000001 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626171+0000", 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136966 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626214+0000", 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2703000000000001e-05 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:88027 9.f e381/376 9:ff7c9408:::benchmark_data_smithi017_339564_object88026:head v 381'5566, mlcod=381'5564)", 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.578906378999999, 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22187700399999999, 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.052 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2099999999999996e-07 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393408+0000", 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153683 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393418+0000", 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0006999999999999e-05 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393424+0000", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4989999999999999e-06 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393492+0000", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7174000000000004e-05 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393511+0000", 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9831999999999999e-05 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615091+0000", 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.221579844 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615131+0000", 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9344000000000001e-05 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.056 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:88023 9.2 e381/376 ondisk, result = 0)", 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393804+0000", 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.578356343999999, 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22105565699999999, 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393804+0000", 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393804+0000", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393804+0000", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9800000000000003e-07 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393809+0000", 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6290000000000001e-06 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393820+0000", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1251e-05 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393827+0000", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3570000000000003e-06 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393906+0000", 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9122000000000006e-05 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393913+0000", 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3429999999999998e-06 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614859+0000", 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.220946157 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88034 9.8 9:16b0f031:::benchmark_data_smithi017_339564_object88033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 14.574286303999999, 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.228835232, 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88034 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397876+0000", 2024-09-13T14:51:18.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.922e-06 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397940+0000", 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4468999999999995e-05 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614828+0000", 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21688771200000001 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614841+0000", 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3155e-05 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622019+0000", 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071777079999999997 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622179+0000", 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016065799999999999 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622695+0000", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051556699999999998 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626221+0000", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035260869999999998 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626332+0000", 2024-09-13T14:51:18.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110741 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626603+0000", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000271363 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626670+0000", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6789e-05 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626709+0000", 2024-09-13T14:51:18.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9060999999999998e-05 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.1912971470000002, 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30486941899999997, 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.067 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780865+0000", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.629e-06 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780911+0000", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6511999999999998e-05 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780922+0000", 2024-09-13T14:51:18.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1454000000000001e-05 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780930+0000", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9019999999999999e-06 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781098+0000", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016806699999999999 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781253+0000", 2024-09-13T14:51:18.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015428100000000001 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781490+0000", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023734 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789140+0000", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076497320000000002 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800223+0000", 2024-09-13T14:51:18.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011082834999999999 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085715+0000", 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28549267499999997 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085732+0000", 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6991999999999999e-05 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.2 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.1867589239999994, 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30060562000000002, 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.073 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785402+0000", 2024-09-13T14:51:18.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8100000000000003e-07 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785421+0000", 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9623000000000002e-05 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785429+0000", 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785435+0000", 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789170+0000", 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037350439999999999 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789333+0000", 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162893 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789491+0000", 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015807199999999999 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797726+0000", 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082348890000000004 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805320+0000", 2024-09-13T14:51:18.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075942850000000001 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085994+0000", 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28067408599999999 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.086007+0000", 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2286000000000001e-05 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.2 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.1850714750000009, 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30002994399999999, 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.080 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:51:18.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787090+0000", 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.561e-06 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788801+0000", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001711113 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788814+0000", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3242e-05 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788822+0000", 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7109999999999995e-06 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794251+0000", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054290520000000002 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794407+0000", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015591100000000001 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794559+0000", 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015160600000000001 2024-09-13T14:51:18.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801869+0000", 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007309973 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087105+0000", 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28523667899999999 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087118+0000", 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3096e-05 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.0541699649999998, 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90794274900000005, 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:51:18.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917993+0000", 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5560000000000002e-06 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918027+0000", 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4338000000000002e-05 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918056+0000", 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9411e-05 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918071+0000", 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4154999999999999e-05 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087146+0000", 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16907546600000001 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087156+0000", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0069e-05 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825063+0000", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.737906583 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825933+0000", 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087017099999999999 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.090 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.12 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.0540631349999998, 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92020213900000003, 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:51:18.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3300000000000003e-07 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918118+0000", 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0727e-05 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0434999999999999e-05 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918134+0000", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5589999999999996e-06 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087181+0000", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16904727899999999 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087188+0000", 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7519999999999999e-06 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838221+0000", 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.751033124 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838299+0000", 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7830000000000005e-05 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.095 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.12 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.0540188649999997, 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92069096500000003, 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918142+0000", 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6800000000000001e-07 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918175+0000", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3642000000000001e-05 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918182+0000", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3920000000000002e-06 2024-09-13T14:51:18.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087211+0000", 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16902355999999999 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087217+0000", 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.835e-06 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838755+0000", 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.75153752500000004 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838832+0000", 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7414000000000001e-05 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95200 9.18 9:1838bbdd:::benchmark_data_smithi017_339564_object95199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:51:18.100 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 7.7840643189999996, 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65754950099999998, 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95200 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:51:18.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188098+0000", 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4040000000000002e-06 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:51:18.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188129+0000", 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1032999999999999e-05 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188155+0000", 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6322999999999999e-05 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188169+0000", 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3512999999999999e-05 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188244+0000", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5078000000000002e-05 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188258+0000", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.347e-05 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839583+0000", 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65132497099999997 2024-09-13T14:51:18.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839664+0000", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1942999999999998e-05 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839771+0000", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010627 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842419+0000", 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026485020000000001 2024-09-13T14:51:18.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842538+0000", 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119218 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845548+0000", 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030097520000000001 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:51:18.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845600+0000", 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2218e-05 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845645+0000", 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4805999999999998e-05 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:51:18.107 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:51:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:18.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:17 smithi189 ceph-mon[90992]: osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:51:18.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:17 smithi189 ceph-mon[90992]: 9.12 scrub starts 2024-09-13T14:51:18.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:17 smithi189 ceph-mon[90992]: pgmap v1300: 109 pgs: 2 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:18.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:51:18.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:51:18.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:18.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:51:18.361 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:18.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:51:18.395 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:18.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:51:18.435 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:18.467 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:51:18.467 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:18.501 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:51:18.502 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:18.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.531 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:51:18.531 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:18.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:51:18.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:18.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:18.668 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:51:18.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:18.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 100487 100471 95.113 13.9688 0.0042913 0.0104756 2024-09-13T14:51:18.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:18.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.005 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:19.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.146 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:51:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[103001]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/750395168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[107556]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/750395168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:19.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:51:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:18 smithi189 ceph-mon[90992]: osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:51:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/750395168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.284 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:19.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.419 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.520 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.566 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:19.703 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:19.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 101360 101344 94.5079 54.5625 0.0100792 0.0105743 2024-09-13T14:51:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:19.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:20.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: pgmap v1303: 109 pgs: 3 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 105 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:20.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: pgmap v1303: 109 pgs: 3 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 105 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:20.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:19 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:51:20.198 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: pgmap v1303: 109 pgs: 3 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 105 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2024-09-13T14:51:20.215 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:19 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:51:20.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:20.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:20.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:20.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:20.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.554 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:20.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:20.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:20.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:20.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:20.817 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:20.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 102086 102070 93.7854 45.375 0.00856883 0.0106261 2024-09-13T14:51:20.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:20.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:20.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:21.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:21.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:21.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[103001]: 9.12 scrub starts 2024-09-13T14:51:21.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:51:21.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[103001]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:51:21.168 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[107556]: 9.12 scrub starts 2024-09-13T14:51:21.168 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:51:21.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:20 smithi017 ceph-mon[107556]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:51:21.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:20 smithi189 ceph-mon[90992]: 9.12 scrub starts 2024-09-13T14:51:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:51:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:20 smithi189 ceph-mon[90992]: osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:51:21.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.568 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:21.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:21.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:21.770 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:21.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:21.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:21.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 102158 102142 92.4917 4.5 0.00478658 0.010625 2024-09-13T14:51:21.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.982 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:21 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:21.982 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:21 smithi189 ceph-mon[90992]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:51:21.983 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:21 smithi189 ceph-mon[90992]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+clean+scrubbing+deep, 105 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/302355 objects misplaced (3.003%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:21.983 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:21 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:51:22.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:22.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[103001]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[103001]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+clean+scrubbing+deep, 105 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/302355 objects misplaced (3.003%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[107556]: osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:51:22.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[107556]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+clean+scrubbing+deep, 105 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/302355 objects misplaced (3.003%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:22.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:21 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:51:22.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:22.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:22.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:22.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:22.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:22.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:22.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:22.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 102483 102467 91.4607 20.3125 0.00326481 0.0109096 2024-09-13T14:51:22.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:22.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:22.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:23.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:51:23.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:23.137 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:23.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:23.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:23.364 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[107556]: 9.7 deep-scrub ok 2024-09-13T14:51:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:51:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[107556]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[103001]: 9.7 deep-scrub ok 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[103001]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:23 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:23.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.613 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:23.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:23.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:23.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:23 smithi189 ceph-mon[90992]: 9.7 deep-scrub ok 2024-09-13T14:51:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:51:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:23 smithi189 ceph-mon[90992]: osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:51:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:23 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:23.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:23 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:23.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:23.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 102556 102540 90.2371 4.5625 0.00376705 0.0109065 2024-09-13T14:51:23.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:23.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:23.988 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:24.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:24.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:24.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.360 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:24.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:24 smithi189 ceph-mon[90992]: 9.3 deep-scrub ok 2024-09-13T14:51:24.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:24 smithi189 ceph-mon[90992]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:51:24.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:24 smithi189 ceph-mon[90992]: pgmap v1309: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/305880 objects misplaced (2.969%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:24.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:24 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:24.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:24.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:24.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:24.642 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:24.642 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[107556]: 9.3 deep-scrub ok 2024-09-13T14:51:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[107556]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:51:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[107556]: pgmap v1309: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/305880 objects misplaced (2.969%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[103001]: 9.3 deep-scrub ok 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[103001]: osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[103001]: pgmap v1309: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 9081/305880 objects misplaced (2.969%); 896 KiB/s, 13 objects/s recovering 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:24 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:24.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:24.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:24.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 102556 102540 88.984 0 - 0.0109065 2024-09-13T14:51:24.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:24.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:24.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:25.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:25.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:25.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:25.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:51:25.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:25 smithi017 ceph-mon[103001]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:51:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:25 smithi017 ceph-mon[107556]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:51:25.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:25.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:25 smithi189 ceph-mon[90992]: osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:51:25.818 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 102881 102865 88.0435 10.1562 0.0163133 0.0113526 2024-09-13T14:51:25.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.882 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.912 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:25.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:26.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:26.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.173 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:26.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:26.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:26.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:26.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:26.554 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:26 smithi189 ceph-mon[90992]: pgmap v1311: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/307482 objects degraded (2.121%); 8286/307482 objects misplaced (2.695%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:51:26.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.610 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:26 smithi017 ceph-mon[103001]: pgmap v1311: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/307482 objects degraded (2.121%); 8286/307482 objects misplaced (2.695%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:51:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:26 smithi017 ceph-mon[107556]: pgmap v1311: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/307482 objects degraded (2.121%); 8286/307482 objects misplaced (2.695%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:51:26.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 103904 103888 87.7177 63.9375 0.0199588 0.0113946 2024-09-13T14:51:26.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:26.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:26.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:27.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:27.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:27.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:27.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:27.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:27.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:27.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:27 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 6522/307482 objects degraded (2.121%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:27.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:27.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:27.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:27.682 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:27.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:27.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:27] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-13T14:51:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 6522/307482 objects degraded (2.121%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 6522/307482 objects degraded (2.121%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:27.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 104905 104889 87.3823 62.5625 0.00741443 0.0114361 2024-09-13T14:51:27.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:28.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:28.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:28.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.308 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:28.345 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:28.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:28.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:28 smithi017 ceph-mon[103001]: pgmap v1312: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/308001 objects degraded (2.118%); 8286/308001 objects misplaced (2.690%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:28 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:28 smithi017 ceph-mon[107556]: pgmap v1312: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/308001 objects degraded (2.118%); 8286/308001 objects misplaced (2.690%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:28 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:28.721 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:28.736 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:28 smithi189 ceph-mon[90992]: pgmap v1312: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing+deep, 105 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 6522/308001 objects degraded (2.118%); 8286/308001 objects misplaced (2.690%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:28.736 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:28 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:28.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:28.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 105879 105863 87.0335 60.875 0.0135526 0.0114842 2024-09-13T14:51:28.880 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:28.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:29.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:29.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:29.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:29.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:29.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:29.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:29.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:29.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:29.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:29.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:29.738 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:29.738 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:29.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:29.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 106855 106839 86.6954 61 0.0431816 0.0115299 2024-09-13T14:51:29.875 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:29.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:30.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:30.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:30.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.299 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:30.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:30.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:30 smithi189 ceph-mon[90992]: pgmap v1313: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 6522/313734 objects degraded (2.079%); 14456/313734 objects misplaced (4.608%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:51:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:30.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:30.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:30.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:30 smithi017 ceph-mon[107556]: pgmap v1313: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 6522/313734 objects degraded (2.079%); 14456/313734 objects misplaced (4.608%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:51:30.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:30 smithi017 ceph-mon[103001]: pgmap v1313: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 6522/313734 objects degraded (2.079%); 14456/313734 objects misplaced (4.608%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:51:30.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:30.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:30.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 107912 107896 86.4308 66.0625 0.00480141 0.0115633 2024-09-13T14:51:30.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:30.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:31.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:31.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:51:31.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:31.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:31.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:31.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.423 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:31.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:31.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:31.597 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:31.623 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:31 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6522/318885 objects degraded (2.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:31 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6522/318885 objects degraded (2.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:31.725 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:31.738 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:31 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6522/318885 objects degraded (2.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:31.757 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:31.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 108823 108807 86.0575 56.9375 0.0175041 0.0116147 2024-09-13T14:51:31.859 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:31.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:31.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:32.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:32.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:32.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:32.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:32.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:32.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:32 smithi189 ceph-mon[90992]: pgmap v1314: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s; 6522/318885 objects degraded (2.045%); 14051/318885 objects misplaced (4.406%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:51:32.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:32.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:32 smithi017 ceph-mon[103001]: pgmap v1314: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s; 6522/318885 objects degraded (2.045%); 14051/318885 objects misplaced (4.406%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:51:32.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:32 smithi017 ceph-mon[107556]: pgmap v1314: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s; 6522/318885 objects degraded (2.045%); 14051/318885 objects misplaced (4.406%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:51:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:32.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:51:32.819413+0000 min lat: 0.00120371 max lat: 2.94925 avg lat: 0.0116566 2024-09-13T14:51:32.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:51:32.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 109807 109791 85.7505 61.5 0.00308677 0.0116566 2024-09-13T14:51:32.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:32.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:32.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:32.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:33.089 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:33.119 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:33.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.265 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:33.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.366 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:33.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:33.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.808 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:33.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 110631 110615 85.3277 51.5 0.010454 0.0117139 2024-09-13T14:51:33.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:33.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:33.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:34.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:34.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:34.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:34.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:34.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:34.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.448 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:34.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:34.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:34.819 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 111615 111599 85.037 61.5 0.00995578 0.0117548 2024-09-13T14:51:34.850 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:34.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:34 smithi017 ceph-mon[107556]: pgmap v1315: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6522/325911 objects degraded (2.001%); 13791/325911 objects misplaced (4.232%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:51:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:34 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:34 smithi017 ceph-mon[103001]: pgmap v1315: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6522/325911 objects degraded (2.001%); 13791/325911 objects misplaced (4.232%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:51:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:34 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:34.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:34 smithi189 ceph-mon[90992]: pgmap v1315: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6522/325911 objects degraded (2.001%); 13791/325911 objects misplaced (4.232%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:51:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:34 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:35.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:35.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.139 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:35.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:35.380 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:35.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:35.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:35.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.683 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 112335 112319 84.5547 45 0.0193161 0.0118216 2024-09-13T14:51:35.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:35.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:35.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:36.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.260 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.400 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:36.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:36.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:36.639 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:36.639 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:36.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:36.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:36.703 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:51:36.703 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:51:36.703 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:51:36.787 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 113416 113400 84.3524 67.5625 0.00744692 0.0118496 2024-09-13T14:51:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:36.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:36.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:36 smithi017 ceph-mon[103001]: pgmap v1316: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s; 6522/330780 objects degraded (1.972%); 13395/330780 objects misplaced (4.050%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:51:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:36 smithi017 ceph-mon[107556]: pgmap v1316: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s; 6522/330780 objects degraded (1.972%); 13395/330780 objects misplaced (4.050%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:51:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:36.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:37.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:36 smithi189 ceph-mon[90992]: pgmap v1316: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s; 6522/330780 objects degraded (1.972%); 13395/330780 objects misplaced (4.050%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:51:37.065 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.104 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:37.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:37.279 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:37.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:51:37.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:37.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:37.571 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:51:37.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.675 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:37] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-13T14:51:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:37.780 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:37.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 114455 114439 84.124 64.9375 0.00851995 0.0118814 2024-09-13T14:51:37.822 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:37.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.960 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:37.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:38.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:38.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.263 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:38.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:38.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:38.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:38.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:38.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:38.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[103001]: pgmap v1317: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 30 MiB/s wr, 487 op/s; 6522/334743 objects degraded (1.948%); 13395/334743 objects misplaced (4.002%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:51:38.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:38.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:38.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:38.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 115563 115547 83.951 69.25 0.015679 0.0119067 2024-09-13T14:51:38.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:38.936 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6522/334743 objects degraded (1.948%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[103001]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[107556]: pgmap v1317: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 30 MiB/s wr, 487 op/s; 6522/334743 objects degraded (1.948%); 13395/334743 objects misplaced (4.002%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6522/334743 objects degraded (1.948%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:38 smithi017 ceph-mon[107556]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:51:38.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:38 smithi189 ceph-mon[90992]: pgmap v1317: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 30 MiB/s wr, 487 op/s; 6522/334743 objects degraded (1.948%); 13395/334743 objects misplaced (4.002%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:51:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:38 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6522/334743 objects degraded (1.948%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:38 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1030322519' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:38 smithi189 ceph-mon[90992]: osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:51:39.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:39.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:39.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:39.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:39.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.637 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:39.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:39.814 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:39.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 116639 116623 83.759 67.25 0.0115893 0.0119332 2024-09-13T14:51:39.863 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:39.863 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:39.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:39.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:39.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:40.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:40.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:40.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:40.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:40.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:40.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:40.330 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:40.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:40.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:40.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:40.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:40.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:51:40.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:40.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:51:40.741 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:40.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 117705 117689 83.5642 66.625 0.0183561 0.0119615 2024-09-13T14:51:40.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:40 smithi017 ceph-mon[103001]: pgmap v1319: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 6522/341319 objects degraded (1.911%); 13135/341319 objects misplaced (3.848%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:51:40.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:40 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:40.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:40 smithi017 ceph-mon[107556]: pgmap v1319: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 6522/341319 objects degraded (1.911%); 13135/341319 objects misplaced (3.848%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:51:40.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:40 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:40.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:40 smithi189 ceph-mon[90992]: pgmap v1319: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 6522/341319 objects degraded (1.911%); 13135/341319 objects misplaced (3.848%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:51:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:40 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:41.056 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:41.091 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:41.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:41.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:41.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:41.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:41.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:41.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:41.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:41.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:41.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:41.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:41.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 118653 118637 83.291 59.25 0.0191024 0.0120002 2024-09-13T14:51:41.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:41.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:42.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:42.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:42.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:42.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:42.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:42.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:42.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:42.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:42.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:42 smithi189 ceph-mon[90992]: pgmap v1320: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 6522/346626 objects degraded (1.882%); 12646/346626 objects misplaced (3.648%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:51:42.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:42.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.756 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:42.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:42.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.01 2024-09-13T14:51:42.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 119470 2024-09-13T14:51:42.820 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 82.9561 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 44.4523 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 240.188 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1327 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 711.255 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3843 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0120499 2024-09-13T14:51:42.821 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0482992 2024-09-13T14:51:42.822 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.94925 2024-09-13T14:51:42.822 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00120371 2024-09-13T14:51:42.824 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:51:42.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:42.908 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:51:42.908 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:51:42.908 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T14:51:42.908 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:51:42.908 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:51:42.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T14:51:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:42 smithi017 ceph-mon[103001]: pgmap v1320: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 6522/346626 objects degraded (1.882%); 12646/346626 objects misplaced (3.648%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:51:42.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:42 smithi017 ceph-mon[107556]: pgmap v1320: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 6522/346626 objects degraded (1.882%); 12646/346626 objects misplaced (3.648%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:51:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:42.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:43.026 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:43.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:43.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:43.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:43.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:43.307 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:43.379 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:43.380 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:43.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:43.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:43.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.681 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:51:43.681 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:51:43.681 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:51:43.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:51:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:43.693 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.729 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:43.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:43.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:43.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3990973140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3990973140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:43.967 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3990973140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:44.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:44.095 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:51:44.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:44.182 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:44.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:44.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:44.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:44.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:44.537 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:44.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:44.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.624 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:44.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.736 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:44.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:44.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: pgmap v1321: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 64 MiB/s wr, 1.03k op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6522/356334 objects degraded (1.830%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.842 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: pgmap v1321: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 64 MiB/s wr, 1.03k op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6522/356334 objects degraded (1.830%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:44.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: pgmap v1321: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 64 MiB/s wr, 1.03k op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6522/356334 objects degraded (1.830%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:44.875 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:51:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:44.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:45.160 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:45.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:45.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:45.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:45.471 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:45.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:45.589 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:45.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:45.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:45.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:45.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:45.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:46.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:46.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:46.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:46.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:46.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:46.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:46.695 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.766 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:166672 8.9 8:9ffb48af:::benchmark_data_smithi017_253885_object75331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:57.148932+0000", 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 169.61338562700001, 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.087727102000000001, 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 166672 2024-09-13T14:51:46.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148932+0000", 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148932+0000", 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148931+0000", 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148937+0000", 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.579e-06 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148949+0000", 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1973e-05 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.148954+0000", 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2599999999999996e-06 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.157040+0000", 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080855779999999995 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.157687+0000", 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00064753800000000002 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.157837+0000", 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014930600000000001 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.173883+0000", 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016046272 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185440+0000", 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011556935000000001 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.233699+0000", 2024-09-13T14:51:46.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048258680999999998 2024-09-13T14:51:46.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.233805+0000", 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000106761 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.236659+0000", 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002853581 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.775 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.775 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:166674 8.9 8:9f85390f:::benchmark_data_smithi017_253885_object75333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:57.151216+0000", 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 169.61110106000001, 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.088823930999999995, 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.776 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 166674 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151216+0000", 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.777 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151216+0000", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151215+0000", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151223+0000", 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2730000000000002e-06 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151238+0000", 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.502e-05 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.151245+0000", 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8759999999999999e-06 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.165687+0000", 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01444199 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.165877+0000", 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018959000000000001 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.166043+0000", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016594900000000001 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.174021+0000", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0079787150000000008 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185475+0000", 2024-09-13T14:51:46.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011453670000000001 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.236722+0000", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.051246883999999999 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.236853+0000", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000130817 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.240040+0000", 2024-09-13T14:51:46.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031876700000000001 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:166678 8.9 8:99ec40bb:::benchmark_data_smithi017_253885_object75337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:57.156983+0000", 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 169.605335, 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.086372130000000005, 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.783 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.784 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 166678 2024-09-13T14:51:46.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.784 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.156983+0000", 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.156983+0000", 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.785 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.156981+0000", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.156989+0000", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9789999999999993e-06 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.157007+0000", 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8428e-05 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.157018+0000", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0745e-05 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.174117+0000", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017098889999999999 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.175987+0000", 2024-09-13T14:51:46.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0018700279999999999 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176157+0000", 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016998100000000001 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185676+0000", 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095191549999999996 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185713+0000", 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7478999999999998e-05 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.240265+0000", 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.054551642999999997 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.240374+0000", 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108695 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.789 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.243355+0000", 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002981044 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:166688 8.9 8:97180ae2:::benchmark_data_smithi017_253885_object75347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:57.172203+0000", 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 169.590114177, 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.074370399000000004, 2024-09-13T14:51:46.791 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.791 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.791 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 166688 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.792 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172203+0000", 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172203+0000", 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172201+0000", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172215+0000", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.402e-05 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172240+0000", 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5236999999999999e-05 2024-09-13T14:51:46.794 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.172250+0000", 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0063e-05 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.180810+0000", 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085598500000000008 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.180977+0000", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016718000000000001 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.181128+0000", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000150567 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185733+0000", 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046049999999999997 2024-09-13T14:51:46.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.185907+0000", 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017423199999999999 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.243558+0000", 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.05765087 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.243771+0000", 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000212511 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.246574+0000", 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028031710000000001 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:166691 8.9 8:9d9b411b:::benchmark_data_smithi017_253885_object75350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:57.176447+0000", 2024-09-13T14:51:46.798 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 169.58587050700001, 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.073352333000000006, 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 166691 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176447+0000", 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176447+0000", 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176445+0000", 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176457+0000", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2601e-05 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176472+0000", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4993e-05 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.176481+0000", 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5280000000000003e-06 2024-09-13T14:51:46.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.186024+0000", 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095426720000000003 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.186354+0000", 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033041800000000002 2024-09-13T14:51:46.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.186736+0000", 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00038240300000000002 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.192263+0000", 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055262640000000003 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.192689+0000", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042674899999999998 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.246788+0000", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.054098248000000002 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.246990+0000", 2024-09-13T14:51:46.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00020246399999999999 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:57.249799+0000", 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028091819999999999 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:167587 8.9 8:931b8edc:::benchmark_data_smithi017_253885_object76246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.406164+0000", 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 168.35615317700001, 2024-09-13T14:51:46.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.069053788000000005, 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 167587 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406164+0000", 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406164+0000", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406164+0000", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406167+0000", 2024-09-13T14:51:46.807 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5999999999999998e-06 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406177+0000", 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3090000000000001e-06 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406181+0000", 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3050000000000003e-06 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.415337+0000", 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091556699999999994 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.415537+0000", 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00020071600000000001 2024-09-13T14:51:46.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.415710+0000", 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017269200000000001 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428736+0000", 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013025541 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428742+0000", 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8589999999999999e-06 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.472168+0000", 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.043425465000000003 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.472365+0000", 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019669899999999999 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.475218+0000", 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028534860000000001 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.812 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:167589 8.9 8:9382d3fd:::benchmark_data_smithi017_253885_object76248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.406513+0000", 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 168.35580446500001, 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.094973479999999999, 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 167589 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406513+0000", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406513+0000", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406512+0000", 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:46.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406518+0000", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.524e-06 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406527+0000", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7709999999999995e-06 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.406532+0000", 2024-09-13T14:51:46.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7479999999999999e-06 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428763+0000", 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022230408 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428902+0000", 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013883999999999999 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.429036+0000", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000134976 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.438846+0000", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0098092519999999992 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.438860+0000", 2024-09-13T14:51:46.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3817e-05 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.498833+0000", 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.059973679000000002 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.498960+0000", 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012676899999999999 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.501487+0000", 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025265330000000001 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:167599 8.9 8:9b7f0b61:::benchmark_data_smithi017_253885_object76258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.425674+0000", 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 168.336643159, 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.078742223, 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.819 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 167599 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425674+0000", 2024-09-13T14:51:46.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425674+0000", 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425673+0000", 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425681+0000", 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.515e-06 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425711+0000", 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.012e-05 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.425717+0000", 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0000000000000002e-06 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.433438+0000", 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077211099999999998 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.433749+0000", 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000311174 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.433935+0000", 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000185959 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.443783+0000", 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0098481550000000008 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.443823+0000", 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9391999999999998e-05 2024-09-13T14:51:46.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.501546+0000", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.057723779000000003 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.501660+0000", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000113405 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.504417+0000", 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027568720000000001 2024-09-13T14:51:46.825 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:167604 8.9 8:9b0b29f6:::benchmark_data_smithi017_253885_object76263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:48:58.428016+0000", 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 168.33430178899999, 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.079838110000000004, 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 167604 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428016+0000", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428016+0000", 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428014+0000", 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428022+0000", 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0360000000000006e-06 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428035+0000", 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2680999999999999e-05 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.428041+0000", 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8320000000000002e-06 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.438999+0000", 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010957751 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.439199+0000", 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00020053900000000001 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.439391+0000", 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00019193199999999999 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.443843+0000", 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0044520230000000003 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.444770+0000", 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00092719500000000002 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.504612+0000", 2024-09-13T14:51:46.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.059841775 2024-09-13T14:51:46.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.504828+0000", 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000215861 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:48:58.507854+0000", 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003025944 2024-09-13T14:51:46.837 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:181381 8.9 8:9452603a:::benchmark_data_smithi017_253885_object90040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:16.787686+0000", 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 149.97463147600001, 2024-09-13T14:51:46.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.099269907000000004, 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 181381 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787686+0000", 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787686+0000", 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787684+0000", 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787693+0000", 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9199999999999993e-06 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787709+0000", 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6759e-05 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.787717+0000", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3860000000000001e-06 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.810408+0000", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022690893 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.810744+0000", 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033619999999999999 2024-09-13T14:51:46.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.811136+0000", 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00039190600000000003 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.816098+0000", 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0049616909999999998 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.843 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.830034+0000", 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013936222 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.884964+0000", 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.054930387999999997 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.885034+0000", 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9628000000000005e-05 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.886956+0000", 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019221850000000001 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:181389 8.9 8:97ba820c:::benchmark_data_smithi017_253885_object90048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:16.797215+0000", 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 149.96510227300001, 2024-09-13T14:51:46.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.091967020999999996, 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 181389 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797215+0000", 2024-09-13T14:51:46.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797215+0000", 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797213+0000", 2024-09-13T14:51:46.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797222+0000", 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6430000000000001e-06 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797338+0000", 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000116199 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.797348+0000", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3200000000000006e-06 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.816180+0000", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018832906999999999 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.816509+0000", 2024-09-13T14:51:46.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032854599999999998 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.816927+0000", 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000418188 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.829969+0000", 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013041550000000001 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.834715+0000", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047461780000000002 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.887001+0000", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.052285732000000001 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.887053+0000", 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1984000000000002e-05 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.889182+0000", 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021296539999999999 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.35952.0:181400 8.9 8:9f634b2a:::benchmark_data_smithi017_253885_object90059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:49:16.815391+0000", 2024-09-13T14:51:46.852 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 149.946926684, 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.075886307, 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.35952", 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 181400 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815391+0000", 2024-09-13T14:51:46.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815391+0000", 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815389+0000", 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815398+0000", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5149999999999995e-06 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815412+0000", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3593e-05 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.815425+0000", 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2738999999999999e-05 2024-09-13T14:51:46.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.821253+0000", 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058284540000000003 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.821403+0000", 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014972400000000001 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.821549+0000", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014573900000000001 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.829975+0000", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084265220000000005 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.834849+0000", 2024-09-13T14:51:46.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048734529999999998 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.889250+0000", 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.054401711999999998 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.889324+0000", 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4097999999999994e-05 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:49:16.891277+0000", 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019528 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:87615 9.1 e381/376 9:81ea976d:::benchmark_data_smithi017_339564_object87614:head v 381'5522, mlcod=381'5518)", 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:03.042960+0000", 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 43.719357016000004, 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.085011385999999994, 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.859 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.042960+0000", 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.042960+0000", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.042960+0000", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.042998+0000", 2024-09-13T14:51:46.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8371999999999998e-05 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.043008+0000", 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0332999999999999e-05 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.043014+0000", 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5550000000000002e-06 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.044168+0000", 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011542049999999999 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.044184+0000", 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6192999999999999e-05 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047379+0000", 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003195186 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.127972+0000", 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.080592497999999999 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.865 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:87621 9.b e381/376 9:d2d979e4:::benchmark_data_smithi017_339564_object87620:head v 381'5326, mlcod=381'5324)", 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:03.047359+0000", 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 43.714958666000001, 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084718151000000005, 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047359+0000", 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047359+0000", 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047358+0000", 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047388+0000", 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9884000000000001e-05 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.047399+0000", 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1216000000000001e-05 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.127940+0000", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.080540754000000006 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128025+0000", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5124999999999996e-05 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128044+0000", 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9091000000000001e-05 2024-09-13T14:51:46.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.132035+0000", 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039910630000000004 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.132077+0000", 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2106e-05 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:87626 9.1 e381/376 9:896c0d29:::benchmark_data_smithi017_339564_object87625:head v 381'5523, mlcod=381'5519)", 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:03.050871+0000", 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 43.711446080000002, 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.084397442000000003, 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.050871+0000", 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.050871+0000", 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.050870+0000", 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.050914+0000", 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4147000000000003e-05 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128097+0000", 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.077182430999999996 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128105+0000", 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8429999999999999e-06 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128146+0000", 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0136000000000003e-05 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.128165+0000", 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9633999999999999e-05 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.135227+0000", 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070616350000000001 2024-09-13T14:51:46.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.135269+0000", 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2058999999999998e-05 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:120833 9.b e394/376 9:d425137b:::benchmark_data_smithi017_339564_object1360:head v 394'7439, mlcod=394'7437)", 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:44.231551+0000", 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 2.5307660279999999, 2024-09-13T14:51:46.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020341642, 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231551+0000", 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231551+0000", 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231550+0000", 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231560+0000", 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9580000000000002e-06 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231580+0000", 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9372e-05 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231587+0000", 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.892e-06 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231632+0000", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5015999999999997e-05 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.231653+0000", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0426999999999999e-05 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.251757+0000", 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020104084000000001 2024-09-13T14:51:46.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:44.251893+0000", 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013619899999999999 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:122248 9.b e394/376 9:dfb74a32:::benchmark_data_smithi017_339564_object2775:head v 394'7533, mlcod=394'7530)", 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:45.620684+0000", 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 1.1416333569999999, 2024-09-13T14:51:46.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018889924999999998, 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620684+0000", 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620684+0000", 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620683+0000", 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620693+0000", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8069999999999994e-06 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620708+0000", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4535000000000001e-05 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.620714+0000", 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0510000000000002e-06 2024-09-13T14:51:46.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.621175+0000", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00046088900000000003 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.621218+0000", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3176000000000001e-05 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.639436+0000", 2024-09-13T14:51:46.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018218435000000002 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:45.639574+0000", 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013793900000000001 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:122792 9.b e394/376 9:d2d94c3e:::benchmark_data_smithi017_339564_object3319:head v 394'7563, mlcod=394'7559)", 2024-09-13T14:51:46.886 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:46.139461+0000", 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.622856085, 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022234311999999999, 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139461+0000", 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139461+0000", 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139460+0000", 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139471+0000", 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1012999999999999e-05 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139491+0000", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9868e-05 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.139498+0000", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2219999999999996e-06 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.141431+0000", 2024-09-13T14:51:46.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001932806 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.141596+0000", 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016511999999999999 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.161515+0000", 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019919310999999999 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.161696+0000", 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00018078199999999999 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:122797 9.b e394/376 9:df89d205:::benchmark_data_smithi017_339564_object3324:head v 394'7565, mlcod=394'7560)", 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:46.147363+0000", 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.61495465900000001, 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021257159000000001, 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.891 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147363+0000", 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147363+0000", 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147362+0000", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147370+0000", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5180000000000004e-06 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147382+0000", 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2289e-05 2024-09-13T14:51:46.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.147388+0000", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9800000000000003e-06 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.154006+0000", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0066179960000000001 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.154030+0000", 2024-09-13T14:51:46.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3900000000000002e-05 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.168507+0000", 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014476223999999999 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.168620+0000", 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011346800000000001 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.895 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:123160 9.13 e394/392 9:cf2e4238:::benchmark_data_smithi017_339564_object3687:head v 394'8799, mlcod=381'5097)", 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:46.486154+0000", 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.27616335400000003, 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020133076999999999, 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:46.896 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486154+0000", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486154+0000", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:46.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486154+0000", 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967296 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486157+0000", 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4230000000000001e-06 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486162+0000", 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7500000000000003e-06 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.486165+0000", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.298e-06 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.494857+0000", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0086913059999999993 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.494879+0000", 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2010000000000001e-05 2024-09-13T14:51:46.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.506068+0000", 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011188995 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:46.506287+0000", 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021951300000000001 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:46.900 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:46.901 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:51:46.901 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:46 smithi189 ceph-mon[90992]: pgmap v1322: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 54 MiB/s wr, 866 op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:46.901 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:46 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:46.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:46 smithi017 ceph-mon[103001]: pgmap v1322: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 54 MiB/s wr, 866 op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:46.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:46 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:46.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:46 smithi017 ceph-mon[107556]: pgmap v1322: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 54 MiB/s wr, 866 op/s; 6522/356334 objects degraded (1.830%); 12336/356334 objects misplaced (3.462%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:51:46.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:46 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:46.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:47.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:47.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.486 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:47.499 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:47] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:51:47.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:47.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:47.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:47.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:51:47.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:47.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:47.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:48.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.242 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:48.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.275 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:48.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:48.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:48.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:48.486 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:48.519 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:48.606 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:51:48.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:48.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:48 smithi189 ceph-mon[90992]: pgmap v1323: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.2 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 50 MiB/s wr, 984 op/s; 6354/353301 objects degraded (1.798%); 11853/353301 objects misplaced (3.355%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:51:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:48.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:48.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:48.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:48.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:48 smithi017 ceph-mon[103001]: pgmap v1323: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.2 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 50 MiB/s wr, 984 op/s; 6354/353301 objects degraded (1.798%); 11853/353301 objects misplaced (3.355%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:51:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:48 smithi017 ceph-mon[107556]: pgmap v1323: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.2 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 50 MiB/s wr, 984 op/s; 6354/353301 objects degraded (1.798%); 11853/353301 objects misplaced (3.355%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:51:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:49.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:49.081 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:49.183 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:49.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:49.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.365 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:49.467 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:49.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:49.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:49.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:49.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:49.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[103001]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6354/353301 objects degraded (1.798%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[107556]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:51:49.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6354/353301 objects degraded (1.798%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:49.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.909 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:49 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:49.910 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1039741233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:49.910 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:49 smithi189 ceph-mon[90992]: osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:51:49.910 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6354/353301 objects degraded (1.798%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:49.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:49.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:50.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:50.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:50.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:50.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.427 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:51:50.427 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:51:50.427 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.427 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "MOSDPGPushReply(9.13 395/392 [PushReplyOp(9:cf2360d3:::benchmark_data_smithi017_339564_object84212:head)])", 2024-09-13T14:51:50.427 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.389201+0000", 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.034863137000000002, 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.034898535000000001, 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389201+0000", 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.428 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389201+0000", 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389201+0000", 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389207+0000", 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8200000000000002e-06 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.429 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389218+0000", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1011e-05 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.389223+0000", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5979999999999996e-06 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423786+0000", 2024-09-13T14:51:50.430 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.034562283999999999 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "MOSDPGPushReply(9.13 395/392 [PushReplyOp(9:ce5e74e7:::benchmark_data_smithi017_339564_object7655:head)])", 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.417569+0000", 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0064955439999999998, 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0065858210000000004, 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.431 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417569+0000", 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417569+0000", 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.432 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417568+0000", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417575+0000", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5529999999999998e-06 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417585+0000", 2024-09-13T14:51:50.433 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0042e-05 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.417592+0000", 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9439999999999999e-06 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.434 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:127143 9.3 9:c53ed801:::benchmark_data_smithi017_339564_object7670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.392011+0000", 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.032053348000000002, 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.032190517000000002, 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 127143 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.435 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392011+0000", 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392011+0000", 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.436 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392011+0000", 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392014+0000", 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0369999999999999e-06 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.437 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392021+0000", 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0090000000000001e-06 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.392025+0000", 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7809999999999996e-06 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.438 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.403334+0000", 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011308617999999999 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.403444+0000", 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000110108 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:51:50.439 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.403965+0000", 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00052128799999999996 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423760+0000", 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019794901 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.440 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:127171 9.7 e395/383 9:e1021f29:::benchmark_data_smithi017_339564_object7698:head v 395'8958, mlcod=395'8956)", 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.422072+0000", 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0019926660000000001, 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022044529999999999, 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422072+0000", 2024-09-13T14:51:50.441 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422072+0000", 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422071+0000", 2024-09-13T14:51:50.442 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422077+0000", 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8850000000000004e-06 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422086+0000", 2024-09-13T14:51:50.443 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7600000000000008e-06 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422091+0000", 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0100000000000003e-06 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422119+0000", 2024-09-13T14:51:50.444 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7906999999999999e-05 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422137+0000", 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8165e-05 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.445 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:127165 9.a e395/376 9:5120c3b8:::benchmark_data_smithi017_339564_object7692:head v 395'7910, mlcod=395'7907)", 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.422757+0000", 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0013075129999999999, 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0015729349999999999, 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422757+0000", 2024-09-13T14:51:50.446 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422757+0000", 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422756+0000", 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.447 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422762+0000", 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0239999999999997e-06 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422774+0000", 2024-09-13T14:51:50.448 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.203e-05 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422778+0000", 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9230000000000001e-06 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422903+0000", 2024-09-13T14:51:50.449 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000124828 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.422919+0000", 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6099999999999998e-05 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.450 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:127176 9.c e395/376)", 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.423206+0000", 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.00085821400000000005, 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011713330000000001, 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423206+0000", 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.451 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423206+0000", 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423206+0000", 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.452 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423211+0000", 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2009999999999998e-06 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423220+0000", 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6449999999999994e-06 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.453 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.423224+0000", 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.313e-06 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "MOSDPGPushReply(9.13 395/392 [PushReplyOp(9:cf2360d3:::benchmark_data_smithi017_339564_object84212:head)])", 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.398782+0000", 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.025282900000000001, 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025637176000000001, 2024-09-13T14:51:50.454 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398782+0000", 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398782+0000", 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.455 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398781+0000", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398786+0000", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2249999999999999e-06 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398798+0000", 2024-09-13T14:51:50.456 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1787000000000001e-05 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.398804+0000", 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0190000000000003e-06 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:127128 9.13 9:ce5e74e7:::benchmark_data_smithi017_339564_object7655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.377066+0000", 2024-09-13T14:51:50.457 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.046998946999999999, 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047401631999999999, 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for degraded object", 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 127128 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.458 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377066+0000", 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377066+0000", 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377064+0000", 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:50.459 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377071+0000", 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2200000000000003e-06 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377083+0000", 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2277e-05 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.460 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.377090+0000", 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1680000000000001e-06 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.404297+0000", 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.027207088000000001 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.461 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.462 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for degraded object", 2024-09-13T14:51:50.462 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.405774+0000", 2024-09-13T14:51:50.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014773049999999999 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:127101 9.13 9:ce02fc6c:::benchmark_data_smithi017_339564_object7628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-09-13T14:51:50.463 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.352082+0000", 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.071982127000000007, 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.072452777999999995, 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 127101 2024-09-13T14:51:50.464 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352082+0000", 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352082+0000", 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.465 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352081+0000", 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352086+0000", 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2870000000000004e-06 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.466 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352098+0000", 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1970999999999999e-05 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.352104+0000", 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6389999999999997e-06 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.467 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.359322+0000", 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0072184370000000003 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for degraded object", 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.360107+0000", 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00078439500000000001 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.388224+0000", 2024-09-13T14:51:50.468 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.028117263 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.388323+0000", 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8696000000000005e-05 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.388839+0000", 2024-09-13T14:51:50.469 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000516567 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.405855+0000", 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.017015216 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.405899+0000", 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4657999999999998e-05 2024-09-13T14:51:50.470 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.46712.0:127143 9.3 e395/383)", 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.409847+0000", 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.014218003999999999, 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014775686999999999, 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:51:50.471 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409847+0000", 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409847+0000", 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.472 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409846+0000", 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409850+0000", 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2649999999999998e-06 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.473 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409859+0000", 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9080000000000001e-06 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.409864+0000", 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4410000000000003e-06 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "MOSDPGPushReply(9.13 395/392 [PushReplyOp(9:ce5e74e7:::benchmark_data_smithi017_339564_object7655:head)])", 2024-09-13T14:51:50.474 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:50.410092+0000", 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.013972985, 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014596098, 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410092+0000", 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.475 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410092+0000", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410091+0000", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410095+0000", 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1160000000000001e-06 2024-09-13T14:51:50.476 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.477 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.477 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:51:50.477 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410102+0000", 2024-09-13T14:51:50.477 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.861e-06 2024-09-13T14:51:50.477 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:50.410107+0000", 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5689999999999998e-06 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 11 2024-09-13T14:51:50.478 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:51:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:51:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:51:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:50.510 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.651 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:50.895 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:50.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:50 smithi017 ceph-mon[103001]: pgmap v1325: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.01k op/s; 6354/346008 objects degraded (1.836%); 11275/346008 objects misplaced (3.259%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:51:50.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:50 smithi017 ceph-mon[107556]: pgmap v1325: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.01k op/s; 6354/346008 objects degraded (1.836%); 11275/346008 objects misplaced (3.259%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:51:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:50 smithi189 ceph-mon[90992]: pgmap v1325: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.01k op/s; 6354/346008 objects degraded (1.836%); 11275/346008 objects misplaced (3.259%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:51:51.035 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:51.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:51.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:51.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:51.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.449 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:51.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:51.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:51.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:51.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:51.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:51.765 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:51.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:51:51.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:52.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:52.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:52.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.307 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:52.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:52.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:52.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:52.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:52.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:52.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.729 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:52.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.742 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:52 smithi189 ceph-mon[90992]: pgmap v1326: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 1.00k op/s; 6046/340800 objects degraded (1.774%); 10702/340800 objects misplaced (3.140%); 6.5 MiB/s, 103 objects/s recovering 2024-09-13T14:51:52.742 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:52 smithi189 ceph-mon[90992]: 9.18 scrub starts 2024-09-13T14:51:52.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:52.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:52 smithi017 ceph-mon[103001]: pgmap v1326: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 1.00k op/s; 6046/340800 objects degraded (1.774%); 10702/340800 objects misplaced (3.140%); 6.5 MiB/s, 103 objects/s recovering 2024-09-13T14:51:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:52 smithi017 ceph-mon[103001]: 9.18 scrub starts 2024-09-13T14:51:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:52 smithi017 ceph-mon[107556]: pgmap v1326: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 25 MiB/s wr, 1.00k op/s; 6046/340800 objects degraded (1.774%); 10702/340800 objects misplaced (3.140%); 6.5 MiB/s, 103 objects/s recovering 2024-09-13T14:51:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:52 smithi017 ceph-mon[107556]: 9.18 scrub starts 2024-09-13T14:51:52.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:53.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:53.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:53.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:53.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:53.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:53.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.429 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:53.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:53.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:53.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:53.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:53.707 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:53.809 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:53.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:53.910 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:53 smithi189 ceph-mon[90992]: 9.18 scrub ok 2024-09-13T14:51:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:53 smithi017 ceph-mon[103001]: 9.18 scrub ok 2024-09-13T14:51:53.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:53 smithi017 ceph-mon[107556]: 9.18 scrub ok 2024-09-13T14:51:53.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:53.999 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:54.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.238 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:51:54.239 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:51:54.239 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:51:54.239 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:51:54.239 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:51:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:54.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:54.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:54.633 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:54.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:54.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:54.779 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:54.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:54.907 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:54.918 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:54 smithi189 ceph-mon[90992]: pgmap v1327: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:54.918 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6046/330885 objects degraded (1.827%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:54 smithi017 ceph-mon[103001]: pgmap v1327: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6046/330885 objects degraded (1.827%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:54 smithi017 ceph-mon[107556]: pgmap v1327: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6046/330885 objects degraded (1.827%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:54.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:54.962 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:51:55.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:55.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:55.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:55.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:55.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:55.335 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:55.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:55.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:55.440 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:55.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:55.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:55.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:55.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.809 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:55.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:55.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:56.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:56.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:56.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:51:56.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.386 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.495 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:56.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.660 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:56.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:56.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:56 smithi017 ceph-mon[103001]: pgmap v1328: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:56 smithi017 ceph-mon[107556]: pgmap v1328: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:56 smithi189 ceph-mon[90992]: pgmap v1328: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.7 MiB/s wr, 997 op/s; 6046/330885 objects degraded (1.827%); 10092/330885 objects misplaced (3.050%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T14:51:57.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:57.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:57.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:57.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:57.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.481 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:51:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:51:57] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-13T14:51:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:57.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:57.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:57.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:57.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:57.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:57.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:58.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.048 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:58.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:58.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:51:58.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:58.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:58.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:58.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:58.589 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:58.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.730 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:58.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:58.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:58.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:58 smithi017 ceph-mon[103001]: pgmap v1329: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 5733/323397 objects degraded (1.773%); 9513/323397 objects misplaced (2.942%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:58.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:58 smithi017 ceph-mon[107556]: pgmap v1329: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 5733/323397 objects degraded (1.773%); 9513/323397 objects misplaced (2.942%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:58.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:59.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:58 smithi189 ceph-mon[90992]: pgmap v1329: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 5733/323397 objects degraded (1.773%); 9513/323397 objects misplaced (2.942%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:51:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:59.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:59.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:59.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:59.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:59.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:59.398 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:59.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:59.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:59.667 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:59.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:59.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:59.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:59.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:51:59.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:59.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:51:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5733/323397 objects degraded (1.773%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:59.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:51:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5733/323397 objects degraded (1.773%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:51:59.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:51:59.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:51:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5733/323397 objects degraded (1.773%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:00.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:00.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:00.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.257 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:00.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:00.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.415 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:00.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:00.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:00.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:52:00.818 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:00.818 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:52:00.818 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:00.819 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:52:00.819 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:00.819 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:52:00.819 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:00.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:00 smithi017 ceph-mon[103001]: pgmap v1330: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5733/316554 objects degraded (1.811%); 8841/316554 objects misplaced (2.793%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:52:00.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:00 smithi017 ceph-mon[107556]: pgmap v1330: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5733/316554 objects degraded (1.811%); 8841/316554 objects misplaced (2.793%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:52:00.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:52:00.826 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:00.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:00 smithi189 ceph-mon[90992]: pgmap v1330: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5733/316554 objects degraded (1.811%); 8841/316554 objects misplaced (2.793%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:52:00.847 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:00.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:00.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:00.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:52:00.955 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:00.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:00.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:52:00.990 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.030 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:52:01.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:01.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:01.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:01.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:01.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:01.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:01.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.498 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:01.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:01.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:01.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:01.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:01.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:01.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2910330258' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2910330258' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:01.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2910330258' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:02.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:02.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:02.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:02.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:02.242 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:52:02.243 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:02.243 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-09-13T14:52:02.243 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:52:02.243 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:52:02.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-09-13T14:52:02.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.495 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:02.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:02.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:02.735 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:02 smithi017 ceph-mon[103001]: pgmap v1331: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5420/311874 objects degraded (1.738%); 8262/311874 objects misplaced (2.649%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:52:02.751 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:02 smithi017 ceph-mon[107556]: pgmap v1331: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5420/311874 objects degraded (1.738%); 8262/311874 objects misplaced (2.649%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:52:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.768 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:02.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:02.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:03.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:02 smithi189 ceph-mon[90992]: pgmap v1331: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5420/311874 objects degraded (1.738%); 8262/311874 objects misplaced (2.649%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:52:03.043 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:03.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:03.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:03.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:03.409 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:03.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:03.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:03.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.753 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:03.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.801 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:03.801 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:03.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:03.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:03.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:04.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:04.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:04.130 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:04.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:04.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:04.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:04.563 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.597 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:04.682 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:52:04.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:04.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:04.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:04.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:04.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[103001]: pgmap v1332: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:52:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5420/303201 objects degraded (1.788%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[107556]: pgmap v1332: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:52:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5420/303201 objects degraded (1.788%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:04.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:04 smithi189 ceph-mon[90992]: pgmap v1332: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.7 MiB/s, 139 objects/s recovering 2024-09-13T14:52:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5420/303201 objects degraded (1.788%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:05.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:05.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:05.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:05.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:05.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:05.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:05.538 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.567 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:05.807 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:05.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:05.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:05.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[103001]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:05 smithi017 ceph-mon[107556]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:52:05.967 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:06.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:05 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:05 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/836552475' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:05 smithi189 ceph-mon[90992]: osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:52:06.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:06.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:06.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:06.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:06.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:06.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:06.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:06.368 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:06.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:06.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:06.634 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:06.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:06 smithi017 ceph-mon[103001]: pgmap v1334: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:52:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:06 smithi017 ceph-mon[107556]: pgmap v1334: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:52:07.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:06 smithi189 ceph-mon[90992]: pgmap v1334: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 5420/303201 objects degraded (1.788%); 7604/303201 objects misplaced (2.508%); 8.5 MiB/s, 136 objects/s recovering 2024-09-13T14:52:07.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:07.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.275 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:07.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:07.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:07.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:07.657 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:07] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-13T14:52:07.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:07.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:07.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:08.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:08.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:08.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:08.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:08.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:08.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:08.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:08.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:08.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:08.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:52:08.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:08.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:08 smithi017 ceph-mon[107556]: pgmap v1335: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5134/296070 objects degraded (1.734%); 6998/296070 objects misplaced (2.364%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:08.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:08.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:08.917 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:08 smithi017 ceph-mon[103001]: pgmap v1335: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5134/296070 objects degraded (1.734%); 6998/296070 objects misplaced (2.364%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:08.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:08 smithi189 ceph-mon[90992]: pgmap v1335: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5134/296070 objects degraded (1.734%); 6998/296070 objects misplaced (2.364%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:09.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.100 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:09.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:09.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.371 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:09.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:09.612 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.658 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.760 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:09.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:09.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5134/296070 objects degraded (1.734%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:10.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:10.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:10.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5134/296070 objects degraded (1.734%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5134/296070 objects degraded (1.734%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:10.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:10.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.360 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:10.462 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:10.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:10.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:10.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:10.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:10.874 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.899 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:10.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.001 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:10 smithi189 ceph-mon[90992]: pgmap v1336: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5134/289701 objects degraded (1.772%); 6212/289701 objects misplaced (2.144%); 8.9 MiB/s, 141 objects/s recovering 2024-09-13T14:52:11.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:11.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:10 smithi017 ceph-mon[103001]: pgmap v1336: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5134/289701 objects degraded (1.772%); 6212/289701 objects misplaced (2.144%); 8.9 MiB/s, 141 objects/s recovering 2024-09-13T14:52:11.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:10 smithi017 ceph-mon[107556]: pgmap v1336: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5134/289701 objects degraded (1.772%); 6212/289701 objects misplaced (2.144%); 8.9 MiB/s, 141 objects/s recovering 2024-09-13T14:52:11.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:11.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:11.284 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:11.318 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:11.420 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:11.486 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:52:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-13T14:52:11.486 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:52:11.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:11.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:11.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:11.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:11.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:11.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:12.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:12.116 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.153 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:12.255 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:12.278 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:12.292 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:12.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:12.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:12.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[103001]: pgmap v1337: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4840/284751 objects degraded (1.700%); 5609/284751 objects misplaced (1.970%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:52:12.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:12.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:12.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[107556]: pgmap v1337: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4840/284751 objects degraded (1.700%); 5609/284751 objects misplaced (1.970%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:52:12.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:12.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:12.852 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:12.954 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:12.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:12 smithi189 ceph-mon[90992]: pgmap v1337: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4840/284751 objects degraded (1.700%); 5609/284751 objects misplaced (1.970%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:52:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:13.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:13.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:13.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:13.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:52:13.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:13.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:13.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:13.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:13.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.681 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:13.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:13.725 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:13.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:13.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:14.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:14.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:14.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:14.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:14.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:14.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:14.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:14.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:14.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:14.662 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:14.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:14.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:14.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:14.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:14 smithi189 ceph-mon[90992]: pgmap v1338: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:14 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 4840/274833 objects degraded (1.761%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:15.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:14 smithi017 ceph-mon[103001]: pgmap v1338: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:14 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 4840/274833 objects degraded (1.761%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:14 smithi017 ceph-mon[107556]: pgmap v1338: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:52:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:14 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 4840/274833 objects degraded (1.761%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:15.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:15.241 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:15.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:15.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:15.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:15.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:15.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:15.611 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:15.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:15.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:15.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:15.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:16.055 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:16.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:16.157 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:16.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:16.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:16.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:16.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:16.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:16.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:16.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:16.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:16 smithi017 ceph-mon[103001]: pgmap v1339: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:52:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:16 smithi017 ceph-mon[107556]: pgmap v1339: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:52:16.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:17.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:16 smithi189 ceph-mon[90992]: pgmap v1339: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 4840/274833 objects degraded (1.761%); 4745/274833 objects misplaced (1.727%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:52:17.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:17.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:17.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:17.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:17.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:17.313 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:52:17.313 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:17.313 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-13T14:52:17.313 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:52:17.313 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:52:17.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-13T14:52:17.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:17.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:17.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:17.678 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:17] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-13T14:52:17.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:17.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:17.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:17.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:17.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:18.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:18.057 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:18.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:18.143 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:18.245 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:18.283 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:18.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:18.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:18.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:18.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:52:18.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:18.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:18.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:52:18.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:18.620 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:52:18.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:18.652 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:52:18.653 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:18.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:52:18.687 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:18.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:18.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.718 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:52:18.718 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:18.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:52:18.749 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:18.783 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:52:18.784 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:18.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:18.801 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:18 smithi017 ceph-mon[107556]: pgmap v1340: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/266445 objects degraded (1.696%); 4040/266445 objects misplaced (1.516%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:52:18.801 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/29707640' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:18.801 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:18 smithi017 ceph-mon[103001]: pgmap v1340: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/266445 objects degraded (1.696%); 4040/266445 objects misplaced (1.516%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:52:18.801 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/29707640' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:18.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:18.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:18.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:19.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:18 smithi189 ceph-mon[90992]: pgmap v1340: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/266445 objects degraded (1.696%); 4040/266445 objects misplaced (1.516%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:52:19.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/29707640' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:19.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:19.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:19.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:19.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:19.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:19.535 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:19.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:19.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:19 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 4520/266445 objects degraded (1.696%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:19 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 4520/266445 objects degraded (1.696%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:19.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:19 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 4520/266445 objects degraded (1.696%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:20.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:20.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:20.203 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:20.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:20.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:20.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:20.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:20.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:20.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:20.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:20.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.975 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:20 smithi189 ceph-mon[90992]: pgmap v1341: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/259167 objects degraded (1.744%); 3120/259167 objects misplaced (1.204%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:52:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:20 smithi189 ceph-mon[90992]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:52:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.019 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:21.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:21.019 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:20 smithi017 ceph-mon[107556]: pgmap v1341: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/259167 objects degraded (1.744%); 3120/259167 objects misplaced (1.204%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:52:21.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:20 smithi017 ceph-mon[107556]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:52:21.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:20 smithi017 ceph-mon[103001]: pgmap v1341: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4520/259167 objects degraded (1.744%); 3120/259167 objects misplaced (1.204%); 11 MiB/s, 168 objects/s recovering 2024-09-13T14:52:21.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:20 smithi017 ceph-mon[103001]: osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:52:21.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:21.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:21.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:21.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:21.431 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:21.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:21.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:21.671 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.706 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:21.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:21.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:21.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:21.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:21 smithi017 ceph-mon[103001]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:52:21.989 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:21 smithi017 ceph-mon[103001]: 9.6 scrub starts 2024-09-13T14:52:21.989 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:21 smithi017 ceph-mon[107556]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:52:21.990 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:21 smithi017 ceph-mon[107556]: 9.6 scrub starts 2024-09-13T14:52:22.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:21 smithi189 ceph-mon[90992]: osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:52:22.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:21 smithi189 ceph-mon[90992]: 9.6 scrub starts 2024-09-13T14:52:22.014 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:22.044 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:157439 9.2 e398/383 9:426ff5cf:::benchmark_data_smithi017_339564_object37966:head v 398'8053, mlcod=398'8051)", 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:22.037079+0000", 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0058525690000000002, 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058796960000000002, 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:52:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037079+0000", 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037079+0000", 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037080+0000", 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7499999999999999e-07 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037088+0000", 2024-09-13T14:52:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6629999999999993e-06 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037102+0000", 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4656e-05 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037109+0000", 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5810000000000002e-06 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037553+0000", 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044444800000000001 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037570+0000", 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6362999999999999e-05 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:157447 9.e e398/376 9:75a05a8d:::benchmark_data_smithi017_339564_object37974:head v 398'10042, mlcod=398'10040)", 2024-09-13T14:52:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:22.037687+0000", 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0052451169999999997, 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053265099999999996, 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037687+0000", 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037687+0000", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037687+0000", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6199999999999996e-07 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037697+0000", 2024-09-13T14:52:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8269999999999993e-06 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037709+0000", 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1674e-05 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037716+0000", 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1640000000000004e-06 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037761+0000", 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4526e-05 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.037777+0000", 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6205000000000002e-05 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:157450 9.12 9:4cb4913f:::benchmark_data_smithi017_339564_object37977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:52:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:22.040821+0000", 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0021108350000000001, 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022449800000000002, 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for degraded object", 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 157450 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040821+0000", 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040821+0000", 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040822+0000", 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2839999999999999e-06 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040827+0000", 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6349999999999997e-06 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040839+0000", 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.206e-05 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.040847+0000", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8099999999999998e-06 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.041172+0000", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032552200000000002 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:22.041676+0000", 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050347200000000003 2024-09-13T14:52:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 3 2024-09-13T14:52:22.061 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:52:22.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:22.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:22.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:22.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:22.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:22.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:22.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.546 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:22.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:22.647 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:22.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:22.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:22.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:23.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:23.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:23.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:23.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:22 smithi017 ceph-mon[103001]: pgmap v1344: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4202/253971 objects degraded (1.655%); 2235/253971 objects misplaced (0.880%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:52:23.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:22 smithi017 ceph-mon[103001]: 9.6 scrub ok 2024-09-13T14:52:23.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:22 smithi017 ceph-mon[107556]: pgmap v1344: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4202/253971 objects degraded (1.655%); 2235/253971 objects misplaced (0.880%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:52:23.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:22 smithi017 ceph-mon[107556]: 9.6 scrub ok 2024-09-13T14:52:23.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:23.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:52:23.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:22 smithi189 ceph-mon[90992]: pgmap v1344: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 4202/253971 objects degraded (1.655%); 2235/253971 objects misplaced (0.880%); 10 MiB/s, 163 objects/s recovering 2024-09-13T14:52:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:22 smithi189 ceph-mon[90992]: 9.6 scrub ok 2024-09-13T14:52:23.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:23.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:23.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:23.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:23.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:23.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:23.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:23.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:23.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:23.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:23.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:23.816 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:23.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:23.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:24.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:24.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:24.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:24.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:23 smithi017 ceph-mon[103001]: 9.16 deep-scrub starts 2024-09-13T14:52:24.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:23 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 4202/253971 objects degraded (1.655%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:24.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:23 smithi017 ceph-mon[107556]: 9.16 deep-scrub starts 2024-09-13T14:52:24.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:23 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 4202/253971 objects degraded (1.655%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:24.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:24.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:23 smithi189 ceph-mon[90992]: 9.16 deep-scrub starts 2024-09-13T14:52:24.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:23 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 4202/253971 objects degraded (1.655%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:24.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:24.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:24.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:24.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:24.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:24.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:24.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:24.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:25.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:25.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:24 smithi017 ceph-mon[103001]: pgmap v1345: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 11 MiB/s, 179 objects/s recovering 2024-09-13T14:52:25.162 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:24 smithi017 ceph-mon[107556]: pgmap v1345: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 11 MiB/s, 179 objects/s recovering 2024-09-13T14:52:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:25.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:24 smithi189 ceph-mon[90992]: pgmap v1345: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 11 MiB/s, 179 objects/s recovering 2024-09-13T14:52:25.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.400 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:25.664 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:25.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:25.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:26.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:26.204 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:26.233 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:26.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:26.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.477 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:26.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:26.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:26.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:26.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:26.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:26.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:27.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:26 smithi017 ceph-mon[103001]: pgmap v1346: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:26 smithi017 ceph-mon[107556]: pgmap v1346: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:27.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:27.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:26 smithi189 ceph-mon[90992]: pgmap v1346: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4202/244401 objects degraded (1.719%); 1983/244401 objects misplaced (0.811%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:27.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:27.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:27.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:27.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:27.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:27.604 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:27.640 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:27.656 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:27] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:52:27.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:27.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:28.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:28.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:28.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:28.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:28.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:28.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:28.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:28.498 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:52:28.499 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:52:28.499 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:52:28.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:28.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:28.699 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.735 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.790 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:52:28.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:28.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:28.983 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:29.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:29.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:29.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:29.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[103001]: pgmap v1347: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 3948/237486 objects degraded (1.662%); 848/237486 objects misplaced (0.357%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2123202006' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 3948/237486 objects degraded (1.662%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[107556]: pgmap v1347: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 3948/237486 objects degraded (1.662%); 848/237486 objects misplaced (0.357%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2123202006' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:28 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 3948/237486 objects degraded (1.662%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:28 smithi189 ceph-mon[90992]: pgmap v1347: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 3948/237486 objects degraded (1.662%); 848/237486 objects misplaced (0.357%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:52:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2123202006' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:28 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:28 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 3948/237486 objects degraded (1.662%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:29.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.291 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:29.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:29.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:29.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:29.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:29.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:29.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:29.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:29.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:30.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:30.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:30.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:30.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[103001]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:52:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:30.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:30.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:29 smithi017 ceph-mon[107556]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:52:30.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.156 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:29 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:30.156 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:30.156 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:29 smithi189 ceph-mon[90992]: osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:52:30.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:30.251 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:30.264 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:30.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:30.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:30.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:30.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:30.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:30.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:30.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:30.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:30 smithi017 ceph-mon[103001]: pgmap v1349: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 3948/231900 objects degraded (1.702%); 848/231900 objects misplaced (0.366%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:52:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:30 smithi017 ceph-mon[103001]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:52:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:30 smithi017 ceph-mon[107556]: pgmap v1349: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 3948/231900 objects degraded (1.702%); 848/231900 objects misplaced (0.366%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:52:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:30 smithi017 ceph-mon[107556]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:52:30.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:30 smithi189 ceph-mon[90992]: pgmap v1349: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 3948/231900 objects degraded (1.702%); 848/231900 objects misplaced (0.366%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:52:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:30 smithi189 ceph-mon[90992]: osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:52:31.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:31.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:31.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:31.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:31.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:31.486 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:31.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:31.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:31.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:31.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:31.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:31.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.075 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:32.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:32.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:52:32.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[103001]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:52:32.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[103001]: pgmap v1352: 109 pgs: 1 active+clean+scrubbing+deep, 2 active+recovering+undersized+remapped, 106 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4420/228129 objects misplaced (1.938%); 7.2 MiB/s, 115 objects/s recovering 2024-09-13T14:52:32.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:52:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[107556]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:52:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[107556]: pgmap v1352: 109 pgs: 1 active+clean+scrubbing+deep, 2 active+recovering+undersized+remapped, 106 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4420/228129 objects misplaced (1.938%); 7.2 MiB/s, 115 objects/s recovering 2024-09-13T14:52:32.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:31 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:52:32.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:32.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:31 smithi189 ceph-mon[90992]: osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:52:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:31 smithi189 ceph-mon[90992]: pgmap v1352: 109 pgs: 1 active+clean+scrubbing+deep, 2 active+recovering+undersized+remapped, 106 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4420/228129 objects misplaced (1.938%); 7.2 MiB/s, 115 objects/s recovering 2024-09-13T14:52:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:31 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:52:32.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:32.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:32.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:32.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:32.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:32.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:32.772 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:32.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:32.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:32.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:32 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:52:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:32 smithi189 ceph-mon[90992]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:52:33.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:32 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:52:33.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:32 smithi017 ceph-mon[103001]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:52:33.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:32 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:52:33.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:32 smithi017 ceph-mon[107556]: osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:52:33.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:33.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:33.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:33.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:33.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:33.662 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:33.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:33.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:33.942 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:33.943 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:33.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[103001]: pgmap v1354: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3572/219768 objects misplaced (1.625%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:52:34.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:52:34.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:34.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1950983675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[107556]: pgmap v1354: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3572/219768 objects misplaced (1.625%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:52:34.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:52:34.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:34.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1950983675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:34.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:34.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:34.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:34.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:33 smithi189 ceph-mon[90992]: pgmap v1354: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3572/219768 objects misplaced (1.625%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:52:34.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:33 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:52:34.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:33 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:34.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1950983675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:34.433 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:52:34.433 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:34.433 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-13T14:52:34.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd out 0 2024-09-13T14:52:34.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:34.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:34.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:34.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:34.859 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:34.880 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:34.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:34.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.985 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:52:34.985 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:52:34.985 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:52:34.985 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:34.987 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:35.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:35.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:35.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:34 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:52:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:34 smithi017 ceph-mon[103001]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:52:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:34 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:52:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:34 smithi017 ceph-mon[107556]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:52:35.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:35.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:34 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:52:35.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:34 smithi189 ceph-mon[90992]: osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:52:35.261 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:35.363 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.486 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.631 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:35.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:35.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.125 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:36.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[103001]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:52:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[103001]: pgmap v1357: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 3572/219768 objects misplaced (1.625%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:52:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:52:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[103001]: 9.16 deep-scrub ok 2024-09-13T14:52:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[107556]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:52:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[107556]: pgmap v1357: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 3572/219768 objects misplaced (1.625%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:52:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:52:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:35 smithi017 ceph-mon[107556]: 9.16 deep-scrub ok 2024-09-13T14:52:36.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:35 smithi189 ceph-mon[90992]: osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:52:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:35 smithi189 ceph-mon[90992]: pgmap v1357: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 3572/219768 objects misplaced (1.625%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:52:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:52:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:35 smithi189 ceph-mon[90992]: 9.16 deep-scrub ok 2024-09-13T14:52:36.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.545 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:36.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:36.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:36.776 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:36.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:36.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:36.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:52:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:36 smithi017 ceph-mon[103001]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:52:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:52:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:36 smithi017 ceph-mon[107556]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:52:37.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.055 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:37.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:37.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:52:37.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:36 smithi189 ceph-mon[90992]: osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:52:37.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:37.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:37.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.658 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:37] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:52:37.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:37.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:37.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:37.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:37.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[103001]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:52:37.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[103001]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 2864/215949 objects misplaced (1.326%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:37.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:52:37.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[107556]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:52:37.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[107556]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 2864/215949 objects misplaced (1.326%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:37.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:52:37.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:38.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:38.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:38.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:38.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:38.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:37 smithi189 ceph-mon[90992]: osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:52:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:37 smithi189 ceph-mon[90992]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 2864/215949 objects misplaced (1.326%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:38.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:52:38.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:38.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:38.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:38.557 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:38.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:38.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:38.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:52:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[107556]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:52:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[107556]: 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 404'10612 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[107556]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[103001]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[103001]: 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 404'10612 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:38.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:38 smithi017 ceph-mon[103001]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:52:38.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:39.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:39.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:39.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:52:39.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:38 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:52:39.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:38 smithi189 ceph-mon[90992]: osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:52:39.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:38 smithi189 ceph-mon[90992]: 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 404'10612 2024-09-13T14:52:39.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:38 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:39.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:38 smithi189 ceph-mon[90992]: osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:52:39.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:39.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:39.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:39.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:39.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:39.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:39.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:39.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:39.734 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:39.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:39.805 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:39.814 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:39.873 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:39.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:39.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:40.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:40.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:40.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:40.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:40.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:40.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:40.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:40.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.831 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:40.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:40.877 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:52:40.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:40 smithi189 ceph-mon[90992]: pgmap v1363: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6872/213483 objects misplaced (3.219%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:52:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:40 smithi189 ceph-mon[90992]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:52:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/825261979' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.130 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:41.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[103001]: pgmap v1363: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6872/213483 objects misplaced (3.219%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:52:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[103001]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:52:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/825261979' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[107556]: pgmap v1363: 109 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6872/213483 objects misplaced (3.219%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:52:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[107556]: osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:52:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/825261979' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:41.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:41.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:41.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:41.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:41.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:41.523 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:41.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:41.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:41.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:41.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:41.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:41 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:41 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:41 smithi189 ceph-mon[90992]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:52:42.044 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:42.044 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:42.044 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[103001]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:52:42.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:42.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:42.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:41 smithi017 ceph-mon[107556]: osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:52:42.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:42.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:42.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:42.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:42.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:42.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:42.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:42.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:42.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:42.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.933 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:42 smithi189 ceph-mon[90992]: pgmap v1366: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 2 active+recovering+undersized+remapped, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7382/211452 objects misplaced (3.491%); 15 MiB/s, 245 objects/s recovering 2024-09-13T14:52:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:43.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:43.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[103001]: pgmap v1366: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 2 active+recovering+undersized+remapped, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7382/211452 objects misplaced (3.491%); 15 MiB/s, 245 objects/s recovering 2024-09-13T14:52:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:43.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[107556]: pgmap v1366: 109 pgs: 1 active+remapped+backfill_wait, 1 peering, 2 active+recovering+undersized+remapped, 105 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7382/211452 objects misplaced (3.491%); 15 MiB/s, 245 objects/s recovering 2024-09-13T14:52:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:43.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:43.314 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.349 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:43.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:43.450 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:43.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:43.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:43.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:43.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:43.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:43.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:43.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:44.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:44.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:44.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:43 smithi017 ceph-mon[103001]: pgmap v1367: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:52:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:43 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 7038/205164 objects degraded (3.430%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:43 smithi017 ceph-mon[107556]: pgmap v1367: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:52:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:43 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 7038/205164 objects degraded (3.430%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:43 smithi189 ceph-mon[90992]: pgmap v1367: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 15 MiB/s, 238 objects/s recovering 2024-09-13T14:52:44.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:43 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 7038/205164 objects degraded (3.430%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:44.280 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.318 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:44.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:44.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:44.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:44.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:44.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:45.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:45.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:44 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:45.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:44 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:45.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:44 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:45.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.604 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:45.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:45.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:45.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:45.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:45.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:46.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:46.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:46.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:45 smithi017 ceph-mon[103001]: pgmap v1368: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:52:46.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:45 smithi017 ceph-mon[107556]: pgmap v1368: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:52:46.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:45 smithi189 ceph-mon[90992]: pgmap v1368: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 7038/205164 objects degraded (3.430%); 6938/205164 objects misplaced (3.382%); 11 MiB/s, 178 objects/s recovering 2024-09-13T14:52:46.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:46.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:46.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:46.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:46.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:46.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:46.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:46.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:46.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:46.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:46.940 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:46.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:47.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:47.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:47.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:47.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:47.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.531 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:47.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:47.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.691 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:47.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:47] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-13T14:52:47.793 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:47.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:47.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:47.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:48.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:48.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:48.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:48.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:48.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:48.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:48 smithi017 ceph-mon[107556]: pgmap v1369: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 105 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7038/197901 objects degraded (3.556%); 6189/197901 objects misplaced (3.127%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:48.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:48 smithi017 ceph-mon[103001]: pgmap v1369: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 105 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7038/197901 objects degraded (3.556%); 6189/197901 objects misplaced (3.127%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:48.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:48.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:48 smithi189 ceph-mon[90992]: pgmap v1369: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 105 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7038/197901 objects degraded (3.556%); 6189/197901 objects misplaced (3.127%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:48.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.942 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:48.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:49.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:49.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:52:49.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:49.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:49.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:49.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:50.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:49 smithi017 ceph-mon[103001]: pgmap v1370: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6730/190785 objects degraded (3.528%); 5921/190785 objects misplaced (3.103%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:50.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6730/190785 objects degraded (3.528%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:50.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:49 smithi017 ceph-mon[107556]: pgmap v1370: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6730/190785 objects degraded (3.528%); 5921/190785 objects misplaced (3.103%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:50.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6730/190785 objects degraded (3.528%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:50.089 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:50.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:49 smithi189 ceph-mon[90992]: pgmap v1370: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 105 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6730/190785 objects degraded (3.528%); 5921/190785 objects misplaced (3.103%); 11 MiB/s, 183 objects/s recovering 2024-09-13T14:52:50.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6730/190785 objects degraded (3.528%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:52:50.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:50.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:50.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:50.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:52:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:52:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:50.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:50.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:50.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:50.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:50.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:50.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:51.081 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:51.120 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:51.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:51.211 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:52:51.211 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:52:51.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.42073046, 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32587973199999998, 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.212 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:52:51.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786731+0000", 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2778000000000001e-05 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786747+0000", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6589e-05 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786753+0000", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7810000000000002e-06 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792834+0000", 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060810639999999997 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792996+0000", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016137599999999999 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793158+0000", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162034 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797490+0000", 2024-09-13T14:52:51.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043325070000000002 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805581+0000", 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080903590000000001 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112577+0000", 2024-09-13T14:52:51.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30699643500000001 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112587+0000", 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031e-05 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95184 9.2 e382/376 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head v 382'6052, mlcod=382'6047)", 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:52:51.218 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.40711300300001, 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35791878900000001, 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800326+0000", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0500000000000002e-07 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800353+0000", 2024-09-13T14:52:51.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6665e-05 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800365+0000", 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.205e-05 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800374+0000", 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801552+0000", 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011782439999999999 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801573+0000", 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0863999999999999e-05 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.158244+0000", 2024-09-13T14:52:51.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35667120099999999 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95182 9.0 9:0107e64f:::benchmark_data_smithi017_339564_object95181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.288729248, 2024-09-13T14:52:51.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.87911086900000002, 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95182 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918711+0000", 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5579999999999999e-06 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918751+0000", 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9842999999999998e-05 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918774+0000", 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2779999999999999e-05 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918785+0000", 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112614+0000", 2024-09-13T14:52:51.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19382977600000001 2024-09-13T14:52:51.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112623+0000", 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0620000000000007e-06 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.796000+0000", 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68337630999999999 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.797820+0000", 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00182003 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.231 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.28863237900001, 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88068354999999998, 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.232 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:52:51.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:52:51.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5799999999999999e-07 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919323+0000", 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051669300000000004 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919336+0000", 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3485999999999999e-05 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919344+0000", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112646+0000", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19330140800000001 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112652+0000", 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6499999999999999e-06 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799403+0000", 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68675042600000002 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799489+0000", 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6649000000000003e-05 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99743 9.10 9:09e78ae4:::benchmark_data_smithi017_339564_object99742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295760338999997, 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2203931610000001, 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99743 2024-09-13T14:52:51.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911679+0000", 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3689999999999999e-06 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911728+0000", 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.922e-05 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911741+0000", 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2315e-05 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911746+0000", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8080000000000001e-06 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911801+0000", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4157999999999999e-05 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911808+0000", 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6739999999999997e-06 2024-09-13T14:52:51.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102095+0000", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1902865440000001 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102270+0000", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017495 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102476+0000", 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000205787 2024-09-13T14:52:51.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109826+0000", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073506869999999998 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109842+0000", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.554e-05 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115907+0000", 2024-09-13T14:52:51.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060650799999999996 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129977+0000", 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014069813 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.130088+0000", 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111799 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132071+0000", 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019824170000000002 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99758 9.10 9:0b28f730:::benchmark_data_smithi017_339564_object99757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295676087000004, 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2224807790000001, 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.245 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99758 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911763+0000", 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2e-07 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911826+0000", 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3499999999999999e-05 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911834+0000", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9570000000000002e-06 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911838+0000", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0049999999999998e-06 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911869+0000", 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0925e-05 2024-09-13T14:52:51.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911873+0000", 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9539999999999998e-06 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105698+0000", 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.193825261 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105855+0000", 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015642 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.106033+0000", 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017866900000000001 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109848+0000", 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038150580000000001 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121439+0000", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01159059 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132121+0000", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682087999999999 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132209+0000", 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7552000000000005e-05 2024-09-13T14:52:51.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132324+0000", 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115606 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134243+0000", 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001918674 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99769 9.10 9:081d7eac:::benchmark_data_smithi017_339564_object99768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295589268000001, 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.223936653, 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99769 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:52:51.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8800000000000003e-07 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911912+0000", 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2597999999999999e-05 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911920+0000", 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4309999999999994e-06 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911924+0000", 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1389999999999997e-06 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911955+0000", 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0182e-05 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911958+0000", 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6330000000000002e-06 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.107964+0000", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1960057749999999 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108122+0000", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015787400000000001 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108353+0000", 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023077800000000001 2024-09-13T14:52:51.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115855+0000", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075024380000000002 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121484+0000", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056290380000000003 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132167+0000", 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682957 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134290+0000", 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021226890000000001 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134356+0000", 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5739000000000002e-05 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135785+0000", 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014298939999999999 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99779 9.10 9:0b953e14:::benchmark_data_smithi017_339564_object99778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295502952000007, 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.225690441, 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.260 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99779 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911936+0000", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911991+0000", 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5569999999999998e-05 2024-09-13T14:52:51.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911999+0000", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5089999999999996e-06 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912003+0000", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9319999999999998e-06 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912031+0000", 2024-09-13T14:52:51.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8237000000000001e-05 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912034+0000", 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.444e-06 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110235+0000", 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.198200776 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110443+0000", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000207517 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110617+0000", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017471500000000001 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115921+0000", 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053037340000000001 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135838+0000", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019916776000000001 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135894+0000", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6515999999999999e-05 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136027+0000", 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013228 2024-09-13T14:52:51.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136088+0000", 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1495999999999995e-05 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137626+0000", 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015374570000000001 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99792 9.10 9:0afafc95:::benchmark_data_smithi017_339564_object99791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295424733000004, 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2470098199999999, 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99792 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.268 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912014+0000", 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7599999999999997e-07 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912055+0000", 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0843999999999997e-05 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912062+0000", 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3640000000000001e-06 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912066+0000", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5379999999999998e-06 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912091+0000", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5678999999999999e-05 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912095+0000", 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4240000000000002e-06 2024-09-13T14:52:51.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113399+0000", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2013038380000001 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113589+0000", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019008499999999999 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113800+0000", 2024-09-13T14:52:51.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021092600000000001 2024-09-13T14:52:51.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121388+0000", 2024-09-13T14:52:51.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007588086 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135939+0000", 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014551708 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137677+0000", 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001737527 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156695+0000", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019018576999999998 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156755+0000", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0028000000000002e-05 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159023+0000", 2024-09-13T14:52:51.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00226772 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99812 9.10 9:08133f40:::benchmark_data_smithi017_339564_object99811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295364769000003, 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2497001430000001, 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.276 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99812 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912074+0000", 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7999999999999996e-07 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912114+0000", 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9949e-05 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912121+0000", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0820000000000001e-06 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912124+0000", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4309999999999998e-06 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912148+0000", 2024-09-13T14:52:51.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3368999999999999e-05 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912151+0000", 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.45e-06 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115948+0000", 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.203796654 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116167+0000", 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021876999999999999 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116334+0000", 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000167794 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121395+0000", 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050604029999999998 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135984+0000", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014589289 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137719+0000", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001734684 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159936+0000", 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022217263000000001 2024-09-13T14:52:51.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.160008+0000", 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1608999999999998e-05 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161773+0000", 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001765916 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99825 9.10 9:0c0deca2:::benchmark_data_smithi017_339564_object99824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295306083, 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2555997640000001, 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99825 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:52:51.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912171+0000", 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8818e-05 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912178+0000", 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9840000000000004e-06 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912182+0000", 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7809999999999999e-06 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912230+0000", 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7647e-05 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912240+0000", 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.005e-05 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:52:51.288 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118734+0000", 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2064939020000001 2024-09-13T14:52:51.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118892+0000", 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158796 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.119058+0000", 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000165336 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129912+0000", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010854278 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145102+0000", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015189947000000001 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145183+0000", 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1160000000000002e-05 2024-09-13T14:52:51.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165672+0000", 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020488431000000001 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165735+0000", 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2934000000000004e-05 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.167732+0000", 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00199723 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99867 9.10 9:08792be1:::benchmark_data_smithi017_339564_object99866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295248267999995, 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.258038813, 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.293 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99867 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7399999999999998e-07 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912235+0000", 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4255999999999998e-05 2024-09-13T14:52:51.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912244+0000", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3999999999999998e-06 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912250+0000", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912279+0000", 2024-09-13T14:52:51.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8410000000000001e-05 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912283+0000", 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8190000000000002e-06 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121515+0000", 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2092329669999999 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121672+0000", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015640000000000001 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121848+0000", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017612800000000001 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129919+0000", 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080710069999999998 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145146+0000", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015227021 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145227+0000", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1304000000000006e-05 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168260+0000", 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023032357999999999 2024-09-13T14:52:51.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168415+0000", 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015499000000000001 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170229+0000", 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018139320000000001 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99888 9.10 9:09cf0b05:::benchmark_data_smithi017_339564_object99887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295176909999995, 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.260453754, 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99888 2024-09-13T14:52:51.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912262+0000", 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912307+0000", 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5405999999999999e-05 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912314+0000", 2024-09-13T14:52:51.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:52:51.304 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:51.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3220000000000004e-06 2024-09-13T14:52:51.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912318+0000", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7019999999999999e-06 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912344+0000", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.614e-05 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912348+0000", 2024-09-13T14:52:51.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.597e-06 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.123904+0000", 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.211556031 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124058+0000", 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153933 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124254+0000", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019642300000000001 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140317+0000", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016063137000000002 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152386+0000", 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012068270000000001 2024-09-13T14:52:51.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153192+0000", 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080682099999999999 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170291+0000", 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017098248 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170362+0000", 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1415999999999995e-05 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172715+0000", 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002352809 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.309 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99894 9.10 9:0e8bfb10:::benchmark_data_smithi017_339564_object99893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295112040000006, 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2902429099999999, 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99894 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4000000000000002e-07 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912366+0000", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9403999999999997e-05 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912373+0000", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0910000000000004e-06 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912377+0000", 2024-09-13T14:52:51.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6890000000000002e-06 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912402+0000", 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5584000000000001e-05 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912406+0000", 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4419999999999998e-06 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126634+0000", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.214228753 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126788+0000", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015391299999999999 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126969+0000", 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018079000000000001 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145055+0000", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018085981000000001 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152432+0000", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073765209999999996 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153227+0000", 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079486599999999998 2024-09-13T14:52:51.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200627+0000", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047400281000000002 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200692+0000", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5309999999999996e-05 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202569+0000", 2024-09-13T14:52:51.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001876845 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99941 9.10 9:0dcd111c:::benchmark_data_smithi017_339564_object99940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.295053640000006, 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.291876698, 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99941 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912385+0000", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8599999999999998e-07 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912423+0000", 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8223000000000003e-05 2024-09-13T14:52:51.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912430+0000", 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8310000000000003e-06 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912434+0000", 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6679999999999999e-06 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912459+0000", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5049999999999999e-05 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912462+0000", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.439e-06 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137750+0000", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2252876800000001 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137916+0000", 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166047 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.138092+0000", 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017649300000000001 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145062+0000", 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069691930000000003 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153258+0000", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081965000000000007 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156056+0000", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002798067 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202632+0000", 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.046576059000000003 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202696+0000", 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4141999999999997e-05 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204261+0000", 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00156482 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99950 9.10 9:0d57d527:::benchmark_data_smithi017_339564_object99949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.294996835000006, 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2935929129999999, 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99950 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.326 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912442+0000", 2024-09-13T14:52:51.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912480+0000", 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8099999999999998e-05 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912487+0000", 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8990000000000004e-06 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912490+0000", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912526+0000", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5649999999999999e-05 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7969999999999998e-06 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140490+0000", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2279604200000001 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141138+0000", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064823500000000004 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141336+0000", 2024-09-13T14:52:51.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019801899999999999 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145114+0000", 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037779839999999999 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159122+0000", 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014007400999999999 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.163411+0000", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042897459999999997 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204322+0000", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.040910421000000002 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204554+0000", 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023251700000000001 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206034+0000", 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014799290000000001 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99954 9.10 9:0ddee9ee:::benchmark_data_smithi017_339564_object99953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.294940883999999, 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2956523129999999, 2024-09-13T14:52:51.333 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99954 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912498+0000", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.89e-07 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912517+0000", 2024-09-13T14:52:51.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9406999999999999e-05 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912525+0000", 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3699999999999995e-06 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7469999999999999e-06 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912563+0000", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3584999999999998e-05 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912571+0000", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0949999999999996e-06 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143130+0000", 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.230558984 2024-09-13T14:52:51.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143272+0000", 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000142399 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143498+0000", 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022541600000000001 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145240+0000", 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017423510000000001 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161922+0000", 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016681756999999998 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172781+0000", 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010859051 2024-09-13T14:52:51.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206100+0000", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033319357000000001 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206306+0000", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020586899999999999 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208150+0000", 2024-09-13T14:52:51.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018434359999999999 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99965 9.10 9:081c4e71:::benchmark_data_smithi017_339564_object99964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.294901582999998, 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2972334430000001, 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.341 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99965 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0900000000000002e-07 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912588+0000", 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1122000000000003e-05 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912598+0000", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0046e-05 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912604+0000", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9909999999999999e-06 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912650+0000", 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6013000000000001e-05 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912657+0000", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7429999999999996e-06 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145268+0000", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2326112730000001 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145425+0000", 2024-09-13T14:52:51.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156794 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145584+0000", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158839 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150428+0000", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048441869999999998 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172958+0000", 2024-09-13T14:52:51.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022530171000000002 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178611+0000", 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056524990000000001 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208220+0000", 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029609139999999999 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208377+0000", 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015774100000000001 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209770+0000", 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001392475 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100028 9.10 9:0bded79a:::benchmark_data_smithi017_339564_object100027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.348 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.294820767000004, 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.300064551, 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100028 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912618+0000", 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912662+0000", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4375000000000001e-05 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912671+0000", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5550000000000003e-06 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912676+0000", 2024-09-13T14:52:51.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8749999999999999e-06 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912725+0000", 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9576000000000001e-05 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912733+0000", 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4150000000000002e-06 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148624+0000", 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2358912049999999 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148772+0000", 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147727 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148947+0000", 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175921 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152859+0000", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039114140000000002 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173020+0000", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020160900999999998 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178657+0000", 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005637268 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209824+0000", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.031167007 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.210229+0000", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040503799999999998 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212682+0000", 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024528060000000001 2024-09-13T14:52:51.355 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100040 9.10 9:0f8c354b:::benchmark_data_smithi017_339564_object100039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 94.294753037999996, 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.302457982, 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:52:51.356 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100040 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3000000000000002e-07 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912719+0000", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3429000000000001e-05 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912724+0000", 2024-09-13T14:52:51.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9490000000000004e-06 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912727+0000", 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912758+0000", 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0531000000000003e-05 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:52:51.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912768+0000", 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0879e-05 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150460+0000", 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2376911479999999 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150598+0000", 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013841999999999999 2024-09-13T14:52:51.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150758+0000", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000159479 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152866+0000", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021080199999999999 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173065+0000", 2024-09-13T14:52:51.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020199238000000001 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178693+0000", 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056281609999999996 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212776+0000", 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034082685000000001 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212896+0000", 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120565 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.215143+0000", 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022468229999999998 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:52:51.363 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:52:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:51.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:51.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:51.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:51.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:51.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:51.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:51.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T14:52:51.473+0000 7f0b3f9c1640 -1 mon.a@0(leader).osd e411 definitely_dead 0 2024-09-13T14:52:51.814 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.0. 2024-09-13T14:52:51.850 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.873 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:51.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[107556]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:52:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/953240549' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[107556]: pgmap v1372: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 105 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6730/185991 objects degraded (3.618%); 5099/185991 objects misplaced (2.742%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[107556]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[103001]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/953240549' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[103001]: pgmap v1372: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 105 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6730/185991 objects degraded (3.618%); 5099/185991 objects misplaced (2.742%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:52:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:51 smithi017 ceph-mon[103001]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:52:52.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:52.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:51 smithi189 ceph-mon[90992]: osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:52:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/953240549' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:51 smithi189 ceph-mon[90992]: pgmap v1372: 109 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 105 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 6730/185991 objects degraded (3.618%); 5099/185991 objects misplaced (2.742%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:52:52.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:52:52.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:51 smithi189 ceph-mon[90992]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:52:52.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:52.377 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:52.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:52.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:52.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:52.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:52.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:52.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:52.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:53.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:53.137 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:52:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:52 smithi017 ceph-mon[103001]: osdmap e412: 8 total, 8 up, 7 in 2024-09-13T14:52:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:52:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:52 smithi017 ceph-mon[107556]: osdmap e412: 8 total, 8 up, 7 in 2024-09-13T14:52:53.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:53.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/466658714' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:52:53.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:52 smithi189 ceph-mon[90992]: osdmap e412: 8 total, 8 up, 7 in 2024-09-13T14:52:53.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:53.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:53.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:53.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:53.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:53.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:53.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:53.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:53.870 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:53.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:54.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:54.141 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:54.179 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[103001]: osdmap e413: 8 total, 8 up, 7 in 2024-09-13T14:52:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[103001]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 4635/179376 objects misplaced (2.584%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[103001]: osdmap e414: 8 total, 8 up, 7 in 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[107556]: osdmap e413: 8 total, 8 up, 7 in 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[107556]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 4635/179376 objects misplaced (2.584%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:52:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:53 smithi017 ceph-mon[107556]: osdmap e414: 8 total, 8 up, 7 in 2024-09-13T14:52:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:53 smithi189 ceph-mon[90992]: osdmap e413: 8 total, 8 up, 7 in 2024-09-13T14:52:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:53 smithi189 ceph-mon[90992]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 4635/179376 objects misplaced (2.584%); 10 MiB/s, 166 objects/s recovering 2024-09-13T14:52:54.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:53 smithi189 ceph-mon[90992]: osdmap e414: 8 total, 8 up, 7 in 2024-09-13T14:52:54.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:54.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:54.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:54.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:54.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:54.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:54.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[107556]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 411'11002 2024-09-13T14:52:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[107556]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T14:52:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6730/185991 objects degraded (3.618%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:52:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[107556]: osdmap e415: 8 total, 8 up, 7 in 2024-09-13T14:52:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[103001]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 411'11002 2024-09-13T14:52:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[103001]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T14:52:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6730/185991 objects degraded (3.618%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:52:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:54 smithi017 ceph-mon[103001]: osdmap e415: 8 total, 8 up, 7 in 2024-09-13T14:52:54.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:54.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:54.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:55.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:52:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.107 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:55.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:54 smithi189 ceph-mon[90992]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 411'11002 2024-09-13T14:52:55.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:54 smithi189 ceph-mon[90992]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-09-13T14:52:55.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:54 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6730/185991 objects degraded (3.618%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:52:55.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:54 smithi189 ceph-mon[90992]: osdmap e415: 8 total, 8 up, 7 in 2024-09-13T14:52:55.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:55.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.358 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:55.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.528 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:55.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.647 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:55.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:55.788 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:55.834 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:55.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:56.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:56.132 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:56.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:56.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:56.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:56.350 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:56.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:56.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:56.486 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:56.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:56.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:56 smithi189 ceph-mon[90992]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4635/179376 objects misplaced (2.584%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T14:52:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:56 smithi189 ceph-mon[90992]: osdmap e416: 8 total, 8 up, 7 in 2024-09-13T14:52:57.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:56 smithi017 ceph-mon[103001]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4635/179376 objects misplaced (2.584%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T14:52:57.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:56 smithi017 ceph-mon[103001]: osdmap e416: 8 total, 8 up, 7 in 2024-09-13T14:52:57.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:56 smithi017 ceph-mon[107556]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 3 peering, 1 active+remapped+backfilling, 95 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 4635/179376 objects misplaced (2.584%); 5.1 MiB/s, 81 objects/s recovering 2024-09-13T14:52:57.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:56 smithi017 ceph-mon[107556]: osdmap e416: 8 total, 8 up, 7 in 2024-09-13T14:52:57.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:57.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:57.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:57.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:57.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:52:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:52:57] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.51.0" 2024-09-13T14:52:57.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:57.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:57.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:57.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:57.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:58.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:58.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:58.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:58.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:58.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.342 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:58.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:58.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:58.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:58 smithi189 ceph-mon[90992]: pgmap v1380: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 4 peering, 1 active+remapped+backfilling, 91 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 7437/177792 objects degraded (4.183%); 4635/177792 objects misplaced (2.607%); 5.3 MiB/s, 75 objects/s recovering 2024-09-13T14:52:58.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[103001]: pgmap v1380: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 4 peering, 1 active+remapped+backfilling, 91 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 7437/177792 objects degraded (4.183%); 4635/177792 objects misplaced (2.607%); 5.3 MiB/s, 75 objects/s recovering 2024-09-13T14:52:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:52:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:58 smithi017 ceph-mon[107556]: pgmap v1380: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 8 remapped+peering, 1 activating, 4 peering, 1 active+remapped+backfilling, 91 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 7437/177792 objects degraded (4.183%); 4635/177792 objects misplaced (2.607%); 5.3 MiB/s, 75 objects/s recovering 2024-09-13T14:52:58.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:58.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:58.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:58.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.920 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:59.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:59.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:59.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:59.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:59.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:59.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:59.336 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:59.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:52:59 smithi017 ceph-mon[103001]: Health check failed: Degraded data redundancy: 7437/177792 objects degraded (4.183%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:59.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:52:59 smithi017 ceph-mon[107556]: Health check failed: Degraded data redundancy: 7437/177792 objects degraded (4.183%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:59.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:59.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:59.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:52:59 smithi189 ceph-mon[90992]: Health check failed: Degraded data redundancy: 7437/177792 objects degraded (4.183%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:59.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:59.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:52:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:59.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:52:59.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:00.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:00.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:00.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:00.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:00.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:00.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:00.406 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:00 smithi189 ceph-mon[90992]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 95 active+clean; 3.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27918/170781 objects degraded (16.347%); 7638/170781 objects misplaced (4.472%); 24 MiB/s, 383 objects/s recovering 2024-09-13T14:53:00.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:00.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:00.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:00 smithi017 ceph-mon[103001]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 95 active+clean; 3.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27918/170781 objects degraded (16.347%); 7638/170781 objects misplaced (4.472%); 24 MiB/s, 383 objects/s recovering 2024-09-13T14:53:00.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:00 smithi017 ceph-mon[107556]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 peering, 95 active+clean; 3.5 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 27918/170781 objects degraded (16.347%); 7638/170781 objects misplaced (4.472%); 24 MiB/s, 383 objects/s recovering 2024-09-13T14:53:00.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:00.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:00.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.784 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4] out_osds: [0] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:53:00.784 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:53:00.784 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-13T14:53:00.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd in 0 2024-09-13T14:53:00.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:00.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:00.960 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:01.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:01.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:01.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:01.253 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:01 smithi189 ceph-mon[90992]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T14:53:01.253 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:53:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:53:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:185187 9.7 9:e6579172:::benchmark_data_smithi017_339564_object65714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2024-09-13T14:53:01.253 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.225120+0000", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.026084884999999999, 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.026116364999999999, 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 185187 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225120+0000", 2024-09-13T14:53:01.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225120+0000", 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225118+0000", 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225123+0000", 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2870000000000004e-06 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225132+0000", 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5099999999999998e-06 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225136+0000", 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5889999999999996e-06 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225357+0000", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022047899999999999 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.225472+0000", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000114967 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.226126+0000", 2024-09-13T14:53:01.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00065380899999999999 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.236377+0000", 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010251237 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.236763+0000", 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00038630099999999997 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.258 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:185191 9.7 9:e3d24d99:::benchmark_data_smithi017_339564_object65718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.229694+0000", 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.021510616, 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021628418, 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 185191 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229694+0000", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229694+0000", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229692+0000", 2024-09-13T14:53:01.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229700+0000", 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.943e-06 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229717+0000", 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7197000000000002e-05 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229724+0000", 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9020000000000002e-06 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229757+0000", 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2954999999999999e-05 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.229872+0000", 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011490299999999999 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.230454+0000", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00058183900000000001 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.236769+0000", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0063155090000000004 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.243147+0000", 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0063781640000000004 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.263 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:185196 9.5 9:a3d041ef:::benchmark_data_smithi017_339564_object65723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.238728+0000", 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.012476358, 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012672436, 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 185196 2024-09-13T14:53:01.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238728+0000", 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238728+0000", 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238726+0000", 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238734+0000", 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9780000000000005e-06 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238748+0000", 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.395e-05 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.266 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238755+0000", 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3219999999999998e-06 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238805+0000", 2024-09-13T14:53:01.267 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0377000000000002e-05 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.238974+0000", 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016862999999999999 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239527+0000", 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00055352000000000005 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.268 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.46712.0:185172 9.5 e416/413 ondisk, result = 0)", 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.239831+0000", 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.011373421, 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011636407, 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239831+0000", 2024-09-13T14:53:01.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239831+0000", 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239830+0000", 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239847+0000", 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7156e-05 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239860+0000", 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2394e-05 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.239866+0000", 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0279999999999997e-06 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.248975+0000", 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0091092250000000003 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.248998+0000", 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2699000000000001e-05 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.272 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:185197 9.d e416/376 9:b2859208:::benchmark_data_smithi017_339564_object65724:head v 416'11559, mlcod=416'11557)", 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.240546+0000", 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.010658502, 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010980402, 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240546+0000", 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.273 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240546+0000", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240545+0000", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240551+0000", 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2349999999999999e-06 2024-09-13T14:53:01.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240567+0000", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6223999999999999e-05 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.240573+0000", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.127e-06 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.242431+0000", 2024-09-13T14:53:01.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001858258 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.242454+0000", 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2772000000000002e-05 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:185201 9.a 9:5b7ba97d:::benchmark_data_smithi017_339564_object65728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.244187+0000", 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0070174419999999996, 2024-09-13T14:53:01.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073935900000000002, 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 185201 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244187+0000", 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244187+0000", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244185+0000", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244193+0000", 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5989999999999996e-06 2024-09-13T14:53:01.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244206+0000", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3217000000000001e-05 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244213+0000", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4300000000000003e-06 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244266+0000", 2024-09-13T14:53:01.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3313999999999998e-05 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244396+0000", 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000129605 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.244522+0000", 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012642600000000001 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249837+0000", 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0053148409999999998 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249844+0000", 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5129999999999999e-06 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.46712.0:185196 9.5 e416/413 ondisk, result = 0)", 2024-09-13T14:53:01.281 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.249780+0000", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0014241550000000001, 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001872023, 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249780+0000", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249780+0000", 2024-09-13T14:53:01.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249780+0000", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249784+0000", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1729999999999997e-06 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249798+0000", 2024-09-13T14:53:01.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3637e-05 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249803+0000", 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3120000000000001e-06 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.251587+0000", 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001783787 2024-09-13T14:53:01.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.251608+0000", 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1100000000000001e-05 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.46712.0:185202 9.9 e416/413 9:93fc694a:::benchmark_data_smithi017_339564_object65729:head v 416'8822, mlcod=380'3737)", 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:01.249901+0000", 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0013035049999999999, 2024-09-13T14:53:01.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0018119010000000001, 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249901+0000", 2024-09-13T14:53:01.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249901+0000", 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249900+0000", 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:01.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249909+0000", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7420000000000003e-06 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249925+0000", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6140000000000001e-05 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.249937+0000", 2024-09-13T14:53:01.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2029e-05 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.250016+0000", 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9066000000000005e-05 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:01.250047+0000", 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.061e-05 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.289 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:53:01.290 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 8 2024-09-13T14:53:01.290 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:53:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:01.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:01.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:01 smithi017 ceph-mon[103001]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T14:53:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:01 smithi017 ceph-mon[107556]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-09-13T14:53:01.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:01.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:01.686 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.729 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:01.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:01.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.864 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:02.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:02.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.163 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:53:02.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:53:02.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:02.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:02.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:02.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.550 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:02 smithi189 ceph-mon[90992]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 27918/167187 objects degraded (16.699%); 7638/167187 objects misplaced (4.569%); 19 MiB/s, 302 objects/s recovering 2024-09-13T14:53:02.550 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:02 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:02.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:02 smithi017 ceph-mon[103001]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 27918/167187 objects degraded (16.699%); 7638/167187 objects misplaced (4.569%); 19 MiB/s, 302 objects/s recovering 2024-09-13T14:53:02.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:02 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:02.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:02 smithi017 ceph-mon[107556]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 27918/167187 objects degraded (16.699%); 7638/167187 objects misplaced (4.569%); 19 MiB/s, 302 objects/s recovering 2024-09-13T14:53:02.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:02 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:02.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:02.699 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:02.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:02.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:02.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:02.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:03.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:03.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.145 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:53:03.145 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:03.163 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:03 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:03.163 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:03.163 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:03 smithi189 ceph-mon[90992]: osdmap e417: 8 total, 8 up, 7 in 2024-09-13T14:53:03.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.204 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:53:03.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:03.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[103001]: osdmap e417: 8 total, 8 up, 7 in 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1477212337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1596672902' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[107556]: osdmap e417: 8 total, 8 up, 7 in 2024-09-13T14:53:03.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1477212337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.234 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:03.260 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:53:03.260 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:03.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:53:03.291 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:03.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:53:03.328 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:03.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:53:03.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:03.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:03.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:03.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:53:03.491 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:03.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1477212337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:03.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:03.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:03.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:03.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:03.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:04.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:04.124 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:04.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:04.226 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:04.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:04.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:04.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:04.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:04 smithi017 ceph-mon[103001]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 22 MiB/s, 350 objects/s recovering 2024-09-13T14:53:04.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 27918/167187 objects degraded (16.699%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:04 smithi017 ceph-mon[107556]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 22 MiB/s, 350 objects/s recovering 2024-09-13T14:53:04.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 27918/167187 objects degraded (16.699%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:04.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:04 smithi189 ceph-mon[90992]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 860 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 22 MiB/s, 350 objects/s recovering 2024-09-13T14:53:04.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 27918/167187 objects degraded (16.699%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:04.517 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:04.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:04.653 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:04.700 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:04.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:05.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:05.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:05.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:05.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:05.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:05.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:05.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:05.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:05.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:05.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:05.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:05.766 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:53:05.803 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:05.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:05.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.197 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:06.197 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:06.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:06.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:06.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:06.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:06.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:06.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:06.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:06.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:06.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:06 smithi017 ceph-mon[103001]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 18 MiB/s, 290 objects/s recovering 2024-09-13T14:53:06.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:06 smithi017 ceph-mon[107556]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 18 MiB/s, 290 objects/s recovering 2024-09-13T14:53:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:06.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:06 smithi189 ceph-mon[90992]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 27672/158712 objects degraded (17.435%); 6892/158712 objects misplaced (4.342%); 18 MiB/s, 290 objects/s recovering 2024-09-13T14:53:06.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:06.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:06.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.947 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:06.966 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:07.016 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:53:07.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:07.181 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:07.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:07.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:07.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.498 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:07.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:07.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:07] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.51.0" 2024-09-13T14:53:07.714 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:07.747 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:07.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:07.885 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:07.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:08.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:08.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:08.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:08.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:08.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:08.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:08.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:08.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T14:53:08.074+0000 7f0b3f9c1640 -1 mon.a@0(leader).osd e417 definitely_dead 0 2024-09-13T14:53:08.465 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:08.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:08.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:08.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[103001]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 27672/155337 objects degraded (17.814%); 6892/155337 objects misplaced (4.437%); 17 MiB/s, 278 objects/s recovering 2024-09-13T14:53:08.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2735252384' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[107556]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 27672/155337 objects degraded (17.814%); 6892/155337 objects misplaced (4.437%); 17 MiB/s, 278 objects/s recovering 2024-09-13T14:53:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2735252384' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:08.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:08 smithi189 ceph-mon[90992]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 27672/155337 objects degraded (17.814%); 6892/155337 objects misplaced (4.437%); 17 MiB/s, 278 objects/s recovering 2024-09-13T14:53:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2735252384' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:53:08.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:08.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:08.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:08.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:08.994 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.0. 2024-09-13T14:53:09.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:09.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:09.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:09.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:09.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:09.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:09.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:09.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[103001]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 27672/155337 objects degraded (17.814%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[103001]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:53:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[107556]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:53:09.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 27672/155337 objects degraded (17.814%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:09.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:09 smithi017 ceph-mon[107556]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:53:09.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:09.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:53:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:09 smithi189 ceph-mon[90992]: osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:53:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:09 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 27672/155337 objects degraded (17.814%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:09.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:09 smithi189 ceph-mon[90992]: osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:53:09.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:09.871 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-13T14:53:09.968 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:09.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:10.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:10.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:10.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:10.512 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:10.527 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:10.554 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:10.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:10.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:10.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:10.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:10.920 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:10 smithi017 ceph-mon[103001]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 972 op/s; 27408/146154 objects degraded (18.753%); 6074/146154 objects misplaced (4.156%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:53:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:10 smithi017 ceph-mon[103001]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:53:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:10 smithi017 ceph-mon[107556]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 972 op/s; 27408/146154 objects degraded (18.753%); 6074/146154 objects misplaced (4.156%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:53:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:10 smithi017 ceph-mon[107556]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:53:10.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.058 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.089 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:11.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:11.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:11.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:10 smithi189 ceph-mon[90992]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 972 op/s; 27408/146154 objects degraded (18.753%); 6074/146154 objects misplaced (4.156%); 9.0 MiB/s, 144 objects/s recovering 2024-09-13T14:53:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:10 smithi189 ceph-mon[90992]: osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:53:11.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.451 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.484 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:11.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:11.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:11.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:11.770 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:53:11.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.890 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:11.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:11.992 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:12.028 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:12.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:11 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/80307618' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:11 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/80307618' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.129 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:12.161 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:12.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:11 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/80307618' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:12.263 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:12.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:12.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:12.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:12.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:12.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:12.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:12.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:12.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:12.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:12.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 20746/143238 objects degraded (14.484%); 6074/143238 objects misplaced (4.240%); 5.6 MiB/s, 90 objects/s recovering 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 20746/143238 objects degraded (14.484%); 6074/143238 objects misplaced (4.240%); 5.6 MiB/s, 90 objects/s recovering 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:12.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:12.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:13.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:13.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 20746/143238 objects degraded (14.484%); 6074/143238 objects misplaced (4.240%); 5.6 MiB/s, 90 objects/s recovering 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:13.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:13.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:13.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:13.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:14.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:14.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:14.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 20746/143238 objects degraded (14.484%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:14.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:14.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:14.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:14.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:13 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 20746/143238 objects degraded (14.484%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:14.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:14.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:14.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:13 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 20746/143238 objects degraded (14.484%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:14.259 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:53:14.259 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:53:14.300 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:14.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:53:14.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:14.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.362 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:53:14.363 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:14.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.396 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:53:14.397 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:14.431 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:53:14.431 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:14.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:53:14.463 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:14.493 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:53:14.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:14.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:14.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:14.645 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:14.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:14.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:14.802 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:14.872 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:53:14.872 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:53:14.872 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:53:14.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:53:14.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:53:15.063 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:15.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:14 smithi017 ceph-mon[103001]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 5.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:15.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:14 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2526152331' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:14 smithi017 ceph-mon[107556]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 5.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:15.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:14 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2526152331' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:15.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:15.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:14 smithi189 ceph-mon[90992]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 5.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:15.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:14 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2526152331' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:15.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:15.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.465 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:15.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:15.529 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:15.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:15.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:15.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.943 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.975 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:16.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:16.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:16.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:16.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.538 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:16.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:16.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:16.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:16.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:16.916 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:16.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:16 smithi017 ceph-mon[103001]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:53:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:16 smithi017 ceph-mon[107556]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:53:17.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:17.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:16 smithi189 ceph-mon[90992]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 8666/136203 objects degraded (6.363%); 1298/136203 objects misplaced (0.953%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:53:17.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.355 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:17.402 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:17] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:53:17.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.492 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:17.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:17.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:17.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.813 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:53:17.813 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:53:17.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:88038 9.f 9:f5f70ae4:::benchmark_data_smithi017_339564_object88037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:03.398320+0000", 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 134.409121019, 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.234966218, 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 88038 2024-09-13T14:53:17.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398320+0000", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398320+0000", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398320+0000", 2024-09-13T14:53:17.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398388+0000", 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8202999999999997e-05 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398394+0000", 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4880000000000004e-06 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.398397+0000", 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1719999999999999e-06 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.403031+0000", 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0046340770000000003 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.403133+0000", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101102 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.403264+0000", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000131204 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.405740+0000", 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024763799999999998 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.407381+0000", 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001640484 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.633203+0000", 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.22582192100000001 2024-09-13T14:53:17.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.633252+0000", 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9846999999999999e-05 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:03.633286+0000", 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3861e-05 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95176 9.7 9:e2f344fa:::benchmark_data_smithi017_339564_object95175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 127.889446666, 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89153507300000001, 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95176 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917992+0000", 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:53:17.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918043+0000", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1029999999999998e-05 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918064+0000", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1078e-05 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918073+0000", 2024-09-13T14:53:17.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6060000000000003e-06 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921343+0000", 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003269597 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921356+0000", 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3698e-05 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.824 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809462+0000", 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88810571900000002 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809529+0000", 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7564999999999994e-05 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95188 9.6 9:6261a5c8:::benchmark_data_smithi017_339564_object95187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:17.825 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 127.889348992, 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.90364369099999997, 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95188 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:53:17.826 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6738e-05 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918136+0000", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8599999999999993e-06 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6580000000000001e-06 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943565+0000", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025424162 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943572+0000", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.985e-06 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808380+0000", 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86480780800000001 2024-09-13T14:53:17.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808541+0000", 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000160958 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808712+0000", 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000170958 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.810740+0000", 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002028555 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.813879+0000", 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031388950000000001 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821630+0000", 2024-09-13T14:53:17.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077507640000000003 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821692+0000", 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2106000000000005e-05 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821736+0000", 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3798999999999999e-05 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.832 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95192 9.16 9:6c6762a8:::benchmark_data_smithi017_339564_object95191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 127.88929377300001, 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.917525484, 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95192 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.833 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:53:17.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0748999999999999e-05 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918195+0000", 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.153e-06 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918199+0000", 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.194e-06 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943779+0000", 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025580471 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943784+0000", 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0289999999999996e-06 2024-09-13T14:53:17.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826118+0000", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88233390700000003 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826264+0000", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014587500000000001 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826408+0000", 2024-09-13T14:53:17.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014380600000000001 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828905+0000", 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002496631 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.830686+0000", 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00178167 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835583+0000", 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048965609999999998 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835635+0000", 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2522000000000003e-05 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835673+0000", 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7376999999999998e-05 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.839 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95195 9.16 9:6f91244f:::benchmark_data_smithi017_339564_object95194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 127.88923717900001, 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91756910000000003, 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95195 2024-09-13T14:53:17.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918203+0000", 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918232+0000", 2024-09-13T14:53:17.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8778000000000001e-05 2024-09-13T14:53:17.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918239+0000", 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8190000000000003e-06 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918243+0000", 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1069999999999998e-06 2024-09-13T14:53:17.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943799+0000", 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025555422000000001 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943802+0000", 2024-09-13T14:53:17.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.749e-06 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827614+0000", 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88381141699999999 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827747+0000", 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013306200000000001 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827906+0000", 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159474 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828913+0000", 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001006369 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.831206+0000", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022932680000000002 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835706+0000", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045005210000000004 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835745+0000", 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8803000000000001e-05 2024-09-13T14:53:17.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835773+0000", 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7801e-05 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159201 9.16 9:6c7b6e8d:::benchmark_data_smithi017_339564_object39728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.896071+0000", 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.911370376000001, 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33514567299999998, 2024-09-13T14:53:17.850 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159201 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896071+0000", 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.851 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896071+0000", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896070+0000", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896073+0000", 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3010000000000001e-06 2024-09-13T14:53:17.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896078+0000", 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5600000000000001e-06 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.896082+0000", 2024-09-13T14:53:17.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6969999999999999e-06 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.963781+0000", 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.067699302000000003 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.963890+0000", 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000108458 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.964104+0000", 2024-09-13T14:53:17.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021441799999999999 2024-09-13T14:53:17.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.966362+0000", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002257668 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.982164+0000", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015802444999999998 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231108+0000", 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.248943731 2024-09-13T14:53:17.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231181+0000", 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2596999999999996e-05 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231216+0000", 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5856999999999999e-05 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159207 9.6 9:6788f10f:::benchmark_data_smithi017_339564_object39734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.905078+0000", 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.902363412, 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.326133695, 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159207 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905078+0000", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905078+0000", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905077+0000", 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:53:17.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905081+0000", 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2799999999999997e-06 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905090+0000", 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0839999999999998e-06 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.905095+0000", 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9370000000000003e-06 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.966788+0000", 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.061693293000000003 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.967134+0000", 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034561500000000002 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.967539+0000", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00040458900000000001 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.979858+0000", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012319808 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.984028+0000", 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0041696559999999999 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231077+0000", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24704895199999999 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231171+0000", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3554999999999995e-05 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.231211+0000", 2024-09-13T14:53:17.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0747999999999997e-05 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159211 9.16 9:68037d94:::benchmark_data_smithi017_339564_object39738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.909783+0000", 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.897657834999997, 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.33588135800000002, 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159211 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909783+0000", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909783+0000", 2024-09-13T14:53:17.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909782+0000", 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909788+0000", 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9240000000000004e-06 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909797+0000", 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1600000000000004e-06 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.909802+0000", 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0270000000000003e-06 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.968696+0000", 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.058893599999999997 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.968802+0000", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000106842 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.969259+0000", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00045691599999999997 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.979864+0000", 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010605069999999999 2024-09-13T14:53:17.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.985769+0000", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059042469999999996 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245542+0000", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25977285900000002 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245626+0000", 2024-09-13T14:53:17.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4829999999999999e-05 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245665+0000", 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8194000000000001e-05 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159238 9.16 9:6cf0fcf4:::benchmark_data_smithi017_339564_object39765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.939576+0000", 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.867865088999999, 2024-09-13T14:53:17.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30621261999999999, 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159238 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939576+0000", 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939576+0000", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939574+0000", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939582+0000", 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9480000000000008e-06 2024-09-13T14:53:17.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939596+0000", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3828999999999999e-05 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.939602+0000", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2929999999999997e-06 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.971251+0000", 2024-09-13T14:53:17.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031648952000000001 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.971348+0000", 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7313999999999997e-05 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.971680+0000", 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000331415 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.979870+0000", 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081905610000000007 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.987830+0000", 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0079600039999999997 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245697+0000", 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25786660700000003 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245758+0000", 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0693999999999999e-05 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245789+0000", 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1024e-05 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.877 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159239 9.16 9:6f7bc7d4:::benchmark_data_smithi017_339564_object39766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.940838+0000", 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.866602688999997, 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.30524961900000003, 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159239 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940838+0000", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940838+0000", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940837+0000", 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940842+0000", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1479999999999997e-06 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940853+0000", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1024e-05 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.940860+0000", 2024-09-13T14:53:17.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3339999999999998e-06 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.973790+0000", 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.032930612999999997 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.973909+0000", 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011877 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.974287+0000", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000377599 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.980040+0000", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057533159999999996 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.989606+0000", 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095661530000000009 2024-09-13T14:53:17.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245992+0000", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25638583999999998 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246055+0000", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2471000000000005e-05 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246088+0000", 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3473999999999998e-05 2024-09-13T14:53:17.883 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159269 9.6 9:66d3e9cd:::benchmark_data_smithi017_339564_object39796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.975522+0000", 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.831919376000002, 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.27043708500000002, 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159269 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975522+0000", 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975522+0000", 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975520+0000", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975526+0000", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.891e-06 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975538+0000", 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1377e-05 2024-09-13T14:53:17.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975543+0000", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3319999999999999e-06 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.976623+0000", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0010795539999999999 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.976690+0000", 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7107000000000006e-05 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.977089+0000", 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000399207 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.980049+0000", 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029595989999999998 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.989638+0000", 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0095895909999999997 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245825+0000", 2024-09-13T14:53:17.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:17.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25618653200000002 2024-09-13T14:53:17.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245926+0000", 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000101828 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.245959+0000", 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2305000000000003e-05 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159270 9.16 9:6e59d64c:::benchmark_data_smithi017_339564_object39797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:23.975628+0000", 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.831813011999998, 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.27082022100000003, 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159270 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975628+0000", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975628+0000", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975627+0000", 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975631+0000", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4230000000000001e-06 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975640+0000", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8410000000000006e-06 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.975644+0000", 2024-09-13T14:53:17.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7249999999999997e-06 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.977263+0000", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001618979 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.977369+0000", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000105208 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.977749+0000", 2024-09-13T14:53:17.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00038021300000000001 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.980051+0000", 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023026069999999999 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:23.991111+0000", 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.01105927 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246378+0000", 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25526775899999998 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246416+0000", 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7839999999999997e-05 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246448+0000", 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1961000000000001e-05 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:159287 9.6 9:66502539:::benchmark_data_smithi017_339564_object39814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:24.001284+0000", 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 53.806157051, 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24492340300000001, 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 159287 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.898 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001284+0000", 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001284+0000", 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001283+0000", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001289+0000", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7360000000000004e-06 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001302+0000", 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2468e-05 2024-09-13T14:53:17.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001307+0000", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3589999999999999e-06 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001357+0000", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0334999999999998e-05 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001484+0000", 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012642900000000001 2024-09-13T14:53:17.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.001931+0000", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000447484 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.004052+0000", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021206559999999998 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.004869+0000", 2024-09-13T14:53:17.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00081718700000000003 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246118+0000", 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24124932800000001 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246178+0000", 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9172999999999997e-05 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:24.246207+0000", 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9777999999999998e-05 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179257 9.12 9:4ecfe978:::benchmark_data_smithi017_339564_object59784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.543189152, 2024-09-13T14:53:17.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7357178289999999, 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179257 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264251+0000", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264257+0000", 2024-09-13T14:53:17.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4439999999999996e-06 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264275+0000", 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8155000000000002e-05 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264288+0000", 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2853e-05 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264350+0000", 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1716000000000003e-05 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264366+0000", 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5863999999999999e-05 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818721+0000", 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.554355614 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818727+0000", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9699999999999996e-06 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822922+0000", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0041942770000001 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822928+0000", 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6710000000000002e-06 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935124+0000", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.112195982 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935256+0000", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013175400000000001 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935638+0000", 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000382023 2024-09-13T14:53:17.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943650+0000", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080123120000000006 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967274+0000", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023624030000000001 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.998994+0000", 2024-09-13T14:53:17.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031719232999999999 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999066+0000", 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2358000000000001e-05 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999970+0000", 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00090380899999999999 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179272 9.12 9:4bdebc4a:::benchmark_data_smithi017_339564_object59799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.510470775999998, 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7069958839999999, 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179272 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.914 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296969+0000", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296974+0000", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1889999999999998e-06 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296984+0000", 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0131e-05 2024-09-13T14:53:17.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296990+0000", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.981e-06 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297030+0000", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9907000000000003e-05 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297036+0000", 2024-09-13T14:53:17.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1199999999999999e-06 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818748+0000", 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.52171188899999998 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818754+0000", 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822950+0000", 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004196181 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822956+0000", 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6510000000000004e-06 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.937987+0000", 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.115030368 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938146+0000", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159703 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938563+0000", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041611300000000002 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946223+0000", 2024-09-13T14:53:17.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007660498 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967408+0000", 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021185239000000002 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003056+0000", 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035647757000000002 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003120+0000", 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3798000000000001e-05 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003966+0000", 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00084632100000000003 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179274 9.12 9:4d2975ed:::benchmark_data_smithi017_339564_object59801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.922 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.504340063000001, 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7113344069999998, 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179274 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:53:17.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303104+0000", 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6600000000000001e-06 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303112+0000", 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5519999999999999e-06 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303117+0000", 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7759999999999997e-06 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303166+0000", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9438000000000003e-05 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303173+0000", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3130000000000001e-06 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818774+0000", 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51560055900000001 2024-09-13T14:53:17.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818779+0000", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3959999999999997e-06 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822980+0000", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004200381 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822987+0000", 2024-09-13T14:53:17.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9269999999999999e-06 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.940936+0000", 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1179493199999999 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941074+0000", 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013840599999999999 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941402+0000", 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032734600000000001 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948293+0000", 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068912189999999996 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967530+0000", 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019237275000000002 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013579+0000", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046049113000000003 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013672+0000", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2820999999999998e-05 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014435+0000", 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000763406 2024-09-13T14:53:17.930 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179351 9.12 9:48a19431:::benchmark_data_smithi017_339564_object59878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.365502278000001, 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5735510970000002, 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179351 2024-09-13T14:53:17.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441937+0000", 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441945+0000", 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.763e-06 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441962+0000", 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7119999999999999e-05 2024-09-13T14:53:17.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441967+0000", 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.75e-06 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442014+0000", 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6425999999999999e-05 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442021+0000", 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.126e-06 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818801+0000", 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.37678042 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818807+0000", 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8989999999999998e-06 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823009+0000", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0042012419999999 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823015+0000", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.477e-06 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943680+0000", 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1206647809999999 2024-09-13T14:53:17.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943812+0000", 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132278 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.944158+0000", 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000345869 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951355+0000", 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071972260000000001 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967668+0000", 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016312454000000001 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014476+0000", 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046808529000000001 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014559+0000", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2373999999999994e-05 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015490+0000", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00093137299999999999 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179385 9.12 9:4cb1548a:::benchmark_data_smithi017_339564_object59912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:53:17.939 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.307635347000002, 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5163587010000001, 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179385 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499805+0000", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499809+0000", 2024-09-13T14:53:17.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5070000000000002e-06 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499825+0000", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5942000000000001e-05 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499831+0000", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6790000000000002e-06 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499889+0000", 2024-09-13T14:53:17.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8090000000000001e-05 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499897+0000", 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2649999999999996e-06 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818835+0000", 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31893799299999998 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818873+0000", 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8136000000000002e-05 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823068+0000", 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004194765 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823075+0000", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946257+0000", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1231814090000001 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946386+0000", 2024-09-13T14:53:17.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012880199999999999 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946701+0000", 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000315719 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954875+0000", 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081736080000000006 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967813+0000", 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012937664 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015557+0000", 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047744885000000001 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015636+0000", 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8121000000000006e-05 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016164+0000", 2024-09-13T14:53:17.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00052882700000000005 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179396 9.12 9:4f84ec82:::benchmark_data_smithi017_339564_object59923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 26.279486250000001, 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4892925720000001, 2024-09-13T14:53:17.949 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179396 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527953+0000", 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527959+0000", 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2009999999999996e-06 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527974+0000", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4151999999999999e-05 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527979+0000", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4029999999999998e-06 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528038+0000", 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8718000000000003e-05 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528051+0000", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2835e-05 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818899+0000", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.29084879699999999 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818906+0000", 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2369999999999997e-06 2024-09-13T14:53:17.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823104+0000", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004197929 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823110+0000", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9099999999999999e-06 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948322+0000", 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1252110909999999 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948445+0000", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123237 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948787+0000", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034259500000000001 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954934+0000", 2024-09-13T14:53:17.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061466480000000002 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967970+0000", 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013036443 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016200+0000", 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048229669000000003 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016731+0000", 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053046400000000002 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017247+0000", 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00051684400000000003 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179450 9.12 9:494d5338:::benchmark_data_smithi017_339564_object59977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:53:17.957 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 25.994794244000001, 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2051270810000001, 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179450 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812644+0000", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812653+0000", 2024-09-13T14:53:17.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7709999999999995e-06 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812673+0000", 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9281000000000001e-05 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812681+0000", 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5040000000000002e-06 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812744+0000", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2874999999999997e-05 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812756+0000", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2223e-05 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818929+0000", 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061728369999999996 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818935+0000", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0499999999999997e-06 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823287+0000", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004352226 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823294+0000", 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3570000000000003e-06 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951389+0000", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.12809574 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951538+0000", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014834099999999999 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.952029+0000", 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00049101500000000003 2024-09-13T14:53:17.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.955950+0000", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039217050000000002 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.968099+0000", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012148681999999999 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017285+0000", 2024-09-13T14:53:17.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049186344 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017360+0000", 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4738000000000002e-05 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017774+0000", 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041378699999999999 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.965 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.966 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:53:17.966 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:53:17.966 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:53:17.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:18.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:18.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:18.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:18.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:18.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:18.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:18.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:53:18.730 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:53:18.792 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:18.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:18.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:19.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:18 smithi189 ceph-mon[90992]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 8666/131640 objects degraded (6.583%); 1298/131640 objects misplaced (0.986%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T14:53:19.111 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:18 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 8666/131640 objects degraded (6.583%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:19.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:53:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:18 smithi017 ceph-mon[103001]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 8666/131640 objects degraded (6.583%); 1298/131640 objects misplaced (0.986%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T14:53:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:18 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 8666/131640 objects degraded (6.583%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:18 smithi017 ceph-mon[107556]: pgmap v1395: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 8666/131640 objects degraded (6.583%); 1298/131640 objects misplaced (0.986%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T14:53:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:18 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 8666/131640 objects degraded (6.583%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:19.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.244 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:19.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:19.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:19.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.524 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.798 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:20.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:20.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:20.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:20.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:19 smithi017 ceph-mon[103001]: from='client.36984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:53:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:19 smithi017 ceph-mon[107556]: from='client.36984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:53:20.216 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:19 smithi189 ceph-mon[90992]: from='client.36984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:53:20.217 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:20.266 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1395,"stamp":"2024-09-13T14:53:17.377922+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2878543845,"num_objects":43880,"num_object_clones":0,"num_object_copies":131640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8666,"num_objects_misplaced":1298,"num_objects_unfound":0,"num_objects_dirty":43880,"num_whiteouts":0,"num_read":5969,"num_read_kb":12982,"num_write":147322,"num_write_kb":4632144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14494,"num_bytes_recovered":948180498,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200043,"ondisk_log_size":200043,"up":327,"acting":317,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":348,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15791612,"kb_used_data":9753968,"kb_used_omap":44,"kb_used_meta":6037331,"kb_avail":734169604,"statfs":{"total":767960285184,"available":751789674496,"internally_reserved":0,"allocated":9988063232,"data_stored":9970178855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":6182227191},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,4,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-317063168,"num_objects":-4838,"num_object_clones":0,"num_object_copies":-14514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-18742,"num_objects_misplaced":-4776,"num_objects_unfound":0,"num_objects_dirty":-4838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4838,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":412,"num_bytes_recovered":26279958,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.007522"},"pg_stats":[{"pgid":"9.19","version":"421'9122","reported_seq":9288,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:13.048209+0000","last_change":"2024-09-13T14:53:08.893085+0000","last_active":"2024-09-13T14:53:13.048209+0000","last_peered":"2024-09-13T14:53:13.048209+0000","last_clean":"2024-09-13T14:52:51.417567+0000","last_became_active":"2024-09-13T14:53:08.855489+0000","last_became_peered":"2024-09-13T14:53:08.855489+0000","last_unstale":"2024-09-13T14:53:13.048209+0000","last_undegraded":"2024-09-13T14:53:08.780172+0000","last_fullsized":"2024-09-13T14:53:08.758673+0000","mapping_epoch":418,"log_start":"379'981","ondisk_log_start":"379'981","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"384'6123","last_scrub_stamp":"2024-09-13T14:51:14.011917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:14.011917+0000","objects_scrubbed":3141,"log_size":8141,"log_dups_size":0,"ondisk_log_size":8141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:47:23.474643+0000","scrub_duration":942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94371840,"num_objects":1440,"num_object_clones":0,"num_object_copies":4320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":388,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3017,"num_write_kb":42112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1440}],"blocked_by":[],"up_primary":0,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":729,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.772058+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:53:11.772058+0000","last_peered":"2024-09-13T14:53:11.772058+0000","last_clean":"2024-09-13T14:53:11.772058+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:53:11.772058+0000","last_undegraded":"2024-09-13T14:53:11.772058+0000","last_fullsized":"2024-09-13T14:53:11.772058+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":419,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.801231+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:53:09.801231+0000","last_peered":"2024-09-13T14:53:09.801231+0000","last_clean":"2024-09-13T14:53:09.801231+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:53:09.801231+0000","last_undegraded":"2024-09-13T14:53:09.801231+0000","last_fullsized":"2024-09-13T14:53:09.801231+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.18","version":"421'9106","reported_seq":9299,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:16.140856+0000","last_change":"2024-09-13T14:53:08.848278+0000","last_active":"2024-09-13T14:53:16.140856+0000","last_peered":"2024-09-13T14:53:16.140856+0000","last_clean":"2024-09-13T14:52:51.490489+0000","last_became_active":"2024-09-13T14:53:08.826242+0000","last_became_peered":"2024-09-13T14:53:08.826242+0000","last_unstale":"2024-09-13T14:53:16.140856+0000","last_undegraded":"2024-09-13T14:53:08.779212+0000","last_fullsized":"2024-09-13T14:53:08.758175+0000","mapping_epoch":418,"log_start":"379'1428","ondisk_log_start":"379'1428","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"395'6993","last_scrub_stamp":"2024-09-13T14:51:52.682533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:52.682533+0000","objects_scrubbed":3448,"log_size":7678,"log_dups_size":0,"ondisk_log_size":7678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86835200,"num_objects":1325,"num_object_clones":0,"num_object_copies":3975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2123,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":1325}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":348,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800062+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:53:09.800062+0000","last_peered":"2024-09-13T14:53:09.800062+0000","last_clean":"2024-09-13T14:53:09.800062+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:53:09.800062+0000","last_undegraded":"2024-09-13T14:53:09.800062+0000","last_fullsized":"2024-09-13T14:53:09.800062+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848380+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:53:11.848380+0000","last_peered":"2024-09-13T14:53:11.848380+0000","last_clean":"2024-09-13T14:53:11.848380+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:53:11.848380+0000","last_undegraded":"2024-09-13T14:53:11.848380+0000","last_fullsized":"2024-09-13T14:53:11.848380+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848784+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:53:11.848784+0000","last_peered":"2024-09-13T14:53:11.848784+0000","last_clean":"2024-09-13T14:53:11.848784+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:53:11.848784+0000","last_undegraded":"2024-09-13T14:53:11.848784+0000","last_fullsized":"2024-09-13T14:53:11.848784+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":388,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800975+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:53:09.800975+0000","last_peered":"2024-09-13T14:53:09.800975+0000","last_clean":"2024-09-13T14:53:09.800975+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:53:09.800975+0000","last_undegraded":"2024-09-13T14:53:09.800975+0000","last_fullsized":"2024-09-13T14:53:09.800975+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":729,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771737+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:53:11.771737+0000","last_peered":"2024-09-13T14:53:11.771737+0000","last_clean":"2024-09-13T14:53:11.771737+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:53:11.771737+0000","last_undegraded":"2024-09-13T14:53:11.771737+0000","last_fullsized":"2024-09-13T14:53:11.771737+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":450,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762116+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:53:09.762116+0000","last_peered":"2024-09-13T14:53:09.762116+0000","last_clean":"2024-09-13T14:53:09.762116+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:53:09.762116+0000","last_undegraded":"2024-09-13T14:53:09.762116+0000","last_fullsized":"2024-09-13T14:53:09.762116+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"421'8999","reported_seq":9164,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:13.194039+0000","last_change":"2024-09-13T14:53:08.886950+0000","last_active":"2024-09-13T14:53:13.194039+0000","last_peered":"2024-09-13T14:53:13.194039+0000","last_clean":"2024-09-13T14:52:51.511890+0000","last_became_active":"2024-09-13T14:53:08.809432+0000","last_became_peered":"2024-09-13T14:53:08.809432+0000","last_unstale":"2024-09-13T14:53:13.194039+0000","last_undegraded":"2024-09-13T14:53:08.772740+0000","last_fullsized":"2024-09-13T14:53:08.759901+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'5946","last_scrub_stamp":"2024-09-13T14:51:12.046918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:12.046918+0000","objects_scrubbed":3020,"log_size":8999,"log_dups_size":0,"ondisk_log_size":8999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:16.350781+0000","scrub_duration":1055,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":350,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3070,"num_write_kb":46016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1388}],"blocked_by":[],"up_primary":0,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":442,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.801459+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:53:09.801459+0000","last_peered":"2024-09-13T14:53:09.801459+0000","last_clean":"2024-09-13T14:53:09.801459+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:53:09.801459+0000","last_undegraded":"2024-09-13T14:53:09.801459+0000","last_fullsized":"2024-09-13T14:53:09.801459+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":333,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762033+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:53:09.762033+0000","last_peered":"2024-09-13T14:53:09.762033+0000","last_clean":"2024-09-13T14:53:09.762033+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:53:09.762033+0000","last_undegraded":"2024-09-13T14:53:09.762033+0000","last_fullsized":"2024-09-13T14:53:09.762033+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"421'9293","reported_seq":9452,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:15.564537+0000","last_change":"2024-09-13T14:51:13.993938+0000","last_active":"2024-09-13T14:53:15.564537+0000","last_peered":"2024-09-13T14:53:15.564537+0000","last_clean":"2024-09-13T14:53:15.564537+0000","last_became_active":"2024-09-13T14:51:10.812491+0000","last_became_peered":"2024-09-13T14:51:10.812491+0000","last_unstale":"2024-09-13T14:53:15.564537+0000","last_undegraded":"2024-09-13T14:53:15.564537+0000","last_fullsized":"2024-09-13T14:53:15.564537+0000","mapping_epoch":376,"log_start":"379'2179","ondisk_log_start":"379'2179","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'6102","last_scrub_stamp":"2024-09-13T14:51:13.993754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:13.993754+0000","objects_scrubbed":3194,"log_size":7114,"log_dups_size":0,"ondisk_log_size":7114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:21.971332+0000","scrub_duration":1273,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3207,"num_write_kb":42752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":432,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.801418+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:53:09.801418+0000","last_peered":"2024-09-13T14:53:09.801418+0000","last_clean":"2024-09-13T14:53:09.801418+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:53:09.801418+0000","last_undegraded":"2024-09-13T14:53:09.801418+0000","last_fullsized":"2024-09-13T14:53:09.801418+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763885+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:53:09.763885+0000","last_peered":"2024-09-13T14:53:09.763885+0000","last_clean":"2024-09-13T14:53:09.763885+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:53:09.763885+0000","last_undegraded":"2024-09-13T14:53:09.763885+0000","last_fullsized":"2024-09-13T14:53:09.763885+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"421'11063","reported_seq":11262,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.171484+0000","last_change":"2024-09-13T14:51:22.141139+0000","last_active":"2024-09-13T14:53:13.171484+0000","last_peered":"2024-09-13T14:53:13.171484+0000","last_clean":"2024-09-13T14:53:13.171484+0000","last_became_active":"2024-09-13T14:51:10.806009+0000","last_became_peered":"2024-09-13T14:51:10.806009+0000","last_unstale":"2024-09-13T14:53:13.171484+0000","last_undegraded":"2024-09-13T14:53:13.171484+0000","last_fullsized":"2024-09-13T14:53:13.171484+0000","mapping_epoch":376,"log_start":"380'2554","ondisk_log_start":"380'2554","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"390'8174","last_scrub_stamp":"2024-09-13T14:51:22.141081+0000","last_deep_scrub":"390'8174","last_deep_scrub_stamp":"2024-09-13T14:51:22.141081+0000","last_clean_scrub_stamp":"2024-09-13T14:51:22.141081+0000","objects_scrubbed":3140,"log_size":8509,"log_dups_size":0,"ondisk_log_size":8509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":9,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:14.780928+0000","scrub_duration":8.2597276760000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93913088,"num_objects":1433,"num_object_clones":0,"num_object_copies":4299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2949,"num_write_kb":39744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":386,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.824059+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:53:09.824059+0000","last_peered":"2024-09-13T14:53:09.824059+0000","last_clean":"2024-09-13T14:53:09.824059+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:53:09.824059+0000","last_undegraded":"2024-09-13T14:53:09.824059+0000","last_fullsized":"2024-09-13T14:53:09.824059+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":401,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762254+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:53:09.762254+0000","last_peered":"2024-09-13T14:53:09.762254+0000","last_clean":"2024-09-13T14:53:09.762254+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:53:09.762254+0000","last_undegraded":"2024-09-13T14:53:09.762254+0000","last_fullsized":"2024-09-13T14:53:09.762254+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"421'9000","reported_seq":9174,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.163042+0000","last_change":"2024-09-13T14:52:21.797011+0000","last_active":"2024-09-13T14:53:13.163042+0000","last_peered":"2024-09-13T14:53:13.163042+0000","last_clean":"2024-09-13T14:53:13.163042+0000","last_became_active":"2024-09-13T14:51:10.805031+0000","last_became_peered":"2024-09-13T14:51:10.805031+0000","last_unstale":"2024-09-13T14:53:13.163042+0000","last_undegraded":"2024-09-13T14:53:13.163042+0000","last_fullsized":"2024-09-13T14:53:13.163042+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"398'7880","last_scrub_stamp":"2024-09-13T14:52:21.796918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:52:21.796918+0000","objects_scrubbed":2495,"log_size":9000,"log_dups_size":0,"ondisk_log_size":9000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:09.261845+0000","scrub_duration":0.88756956499999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88932352,"num_objects":1357,"num_object_clones":0,"num_object_copies":4071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1138,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":349,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762722+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:53:09.762722+0000","last_peered":"2024-09-13T14:53:09.762722+0000","last_clean":"2024-09-13T14:53:09.762722+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:53:09.762722+0000","last_undegraded":"2024-09-13T14:53:09.762722+0000","last_fullsized":"2024-09-13T14:53:09.762722+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848732+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:53:11.848732+0000","last_peered":"2024-09-13T14:53:11.848732+0000","last_clean":"2024-09-13T14:53:11.848732+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:53:11.848732+0000","last_undegraded":"2024-09-13T14:53:11.848732+0000","last_fullsized":"2024-09-13T14:53:11.848732+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"421'9199","reported_seq":9357,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:16.035027+0000","last_change":"2024-09-13T14:51:13.806529+0000","last_active":"2024-09-13T14:53:16.035027+0000","last_peered":"2024-09-13T14:53:16.035027+0000","last_clean":"2024-09-13T14:53:16.035027+0000","last_became_active":"2024-09-13T14:51:10.823221+0000","last_became_peered":"2024-09-13T14:51:10.823221+0000","last_unstale":"2024-09-13T14:53:16.035027+0000","last_undegraded":"2024-09-13T14:53:16.035027+0000","last_fullsized":"2024-09-13T14:53:16.035027+0000","mapping_epoch":376,"log_start":"379'1523","ondisk_log_start":"379'1523","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'6103","last_scrub_stamp":"2024-09-13T14:51:13.806371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:13.806371+0000","objects_scrubbed":3087,"log_size":7676,"log_dups_size":0,"ondisk_log_size":7676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:02.325574+0000","scrub_duration":922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88735744,"num_objects":1354,"num_object_clones":0,"num_object_copies":4062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3111,"num_write_kb":44096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":532,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800781+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:53:09.800781+0000","last_peered":"2024-09-13T14:53:09.800781+0000","last_clean":"2024-09-13T14:53:09.800781+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:53:09.800781+0000","last_undegraded":"2024-09-13T14:53:09.800781+0000","last_fullsized":"2024-09-13T14:53:09.800781+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":761,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771400+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:53:11.771400+0000","last_peered":"2024-09-13T14:53:11.771400+0000","last_clean":"2024-09-13T14:53:11.771400+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:53:11.771400+0000","last_undegraded":"2024-09-13T14:53:11.771400+0000","last_fullsized":"2024-09-13T14:53:11.771400+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.764085+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:53:09.764085+0000","last_peered":"2024-09-13T14:53:09.764085+0000","last_clean":"2024-09-13T14:53:09.764085+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:53:09.764085+0000","last_undegraded":"2024-09-13T14:53:09.764085+0000","last_fullsized":"2024-09-13T14:53:09.764085+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"421'9021","reported_seq":9175,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:15.576978+0000","last_change":"2024-09-13T14:51:12.420135+0000","last_active":"2024-09-13T14:53:15.576978+0000","last_peered":"2024-09-13T14:53:15.576978+0000","last_clean":"2024-09-13T14:53:15.576978+0000","last_became_active":"2024-09-13T14:51:10.792031+0000","last_became_peered":"2024-09-13T14:51:10.792031+0000","last_unstale":"2024-09-13T14:53:15.576978+0000","last_undegraded":"2024-09-13T14:53:15.576978+0000","last_fullsized":"2024-09-13T14:53:15.576978+0000","mapping_epoch":376,"log_start":"379'1711","ondisk_log_start":"379'1711","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'5920","last_scrub_stamp":"2024-09-13T14:51:12.419914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:12.419914+0000","objects_scrubbed":2971,"log_size":7310,"log_dups_size":0,"ondisk_log_size":7310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:54.575754+0000","scrub_duration":675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89391104,"num_objects":1364,"num_object_clones":0,"num_object_copies":4092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3113,"num_write_kb":48192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":246,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762102+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:53:09.762102+0000","last_peered":"2024-09-13T14:53:09.762102+0000","last_clean":"2024-09-13T14:53:09.762102+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:53:09.762102+0000","last_undegraded":"2024-09-13T14:53:09.762102+0000","last_fullsized":"2024-09-13T14:53:09.762102+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":422,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762177+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:53:09.762177+0000","last_peered":"2024-09-13T14:53:09.762177+0000","last_clean":"2024-09-13T14:53:09.762177+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:53:09.762177+0000","last_undegraded":"2024-09-13T14:53:09.762177+0000","last_fullsized":"2024-09-13T14:53:09.762177+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":134,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.818504+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:53:09.818504+0000","last_peered":"2024-09-13T14:53:09.818504+0000","last_clean":"2024-09-13T14:53:09.818504+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:53:09.818504+0000","last_undegraded":"2024-09-13T14:53:09.818504+0000","last_fullsized":"2024-09-13T14:53:09.818504+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"421'11379","reported_seq":11582,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.087606+0000","last_change":"2024-09-13T14:51:23.191689+0000","last_active":"2024-09-13T14:53:13.087606+0000","last_peered":"2024-09-13T14:53:13.087606+0000","last_clean":"2024-09-13T14:53:13.087606+0000","last_became_active":"2024-09-13T14:51:10.819753+0000","last_became_peered":"2024-09-13T14:51:10.819753+0000","last_unstale":"2024-09-13T14:53:13.087606+0000","last_undegraded":"2024-09-13T14:53:13.087606+0000","last_fullsized":"2024-09-13T14:53:13.087606+0000","mapping_epoch":376,"log_start":"381'5413","ondisk_log_start":"381'5413","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"391'8451","last_scrub_stamp":"2024-09-13T14:51:23.191632+0000","last_deep_scrub":"391'8451","last_deep_scrub_stamp":"2024-09-13T14:51:23.191632+0000","last_clean_scrub_stamp":"2024-09-13T14:51:23.191632+0000","objects_scrubbed":3238,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":12,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:35.397415+0000","scrub_duration":11.502028351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91553792,"num_objects":1397,"num_object_clones":0,"num_object_copies":4191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2997,"num_write_kb":36992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763728+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:53:09.763728+0000","last_peered":"2024-09-13T14:53:09.763728+0000","last_clean":"2024-09-13T14:53:09.763728+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:53:09.763728+0000","last_undegraded":"2024-09-13T14:53:09.763728+0000","last_fullsized":"2024-09-13T14:53:09.763728+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.818480+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:53:09.818480+0000","last_peered":"2024-09-13T14:53:09.818480+0000","last_clean":"2024-09-13T14:53:09.818480+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:53:09.818480+0000","last_undegraded":"2024-09-13T14:53:09.818480+0000","last_fullsized":"2024-09-13T14:53:09.818480+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"421'9289","reported_seq":9443,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:16.090545+0000","last_change":"2024-09-13T14:51:12.991601+0000","last_active":"2024-09-13T14:53:16.090545+0000","last_peered":"2024-09-13T14:53:16.090545+0000","last_clean":"2024-09-13T14:53:16.090545+0000","last_became_active":"2024-09-13T14:51:10.821711+0000","last_became_peered":"2024-09-13T14:51:10.821711+0000","last_unstale":"2024-09-13T14:53:16.090545+0000","last_undegraded":"2024-09-13T14:53:16.090545+0000","last_fullsized":"2024-09-13T14:53:16.090545+0000","mapping_epoch":376,"log_start":"379'1568","ondisk_log_start":"379'1568","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'6114","last_scrub_stamp":"2024-09-13T14:51:12.990840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:12.990840+0000","objects_scrubbed":3047,"log_size":7721,"log_dups_size":0,"ondisk_log_size":7721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:41.875757+0000","scrub_duration":1127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85262336,"num_objects":1301,"num_object_clones":0,"num_object_copies":3903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3194,"num_write_kb":46336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":239,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762447+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:53:09.762447+0000","last_peered":"2024-09-13T14:53:09.762447+0000","last_clean":"2024-09-13T14:53:09.762447+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:53:09.762447+0000","last_undegraded":"2024-09-13T14:53:09.762447+0000","last_fullsized":"2024-09-13T14:53:09.762447+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":731,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771711+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:53:11.771711+0000","last_peered":"2024-09-13T14:53:11.771711+0000","last_clean":"2024-09-13T14:53:11.771711+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:53:11.771711+0000","last_undegraded":"2024-09-13T14:53:11.771711+0000","last_fullsized":"2024-09-13T14:53:11.771711+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":693,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.780322+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:53:11.780322+0000","last_peered":"2024-09-13T14:53:11.780322+0000","last_clean":"2024-09-13T14:53:11.780322+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:53:11.780322+0000","last_undegraded":"2024-09-13T14:53:11.780322+0000","last_fullsized":"2024-09-13T14:53:11.780322+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"421'9034","reported_seq":9199,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:16.009791+0000","last_change":"2024-09-13T14:53:08.848274+0000","last_active":"2024-09-13T14:53:16.009791+0000","last_peered":"2024-09-13T14:53:16.009791+0000","last_clean":"2024-09-13T14:52:51.598318+0000","last_became_active":"2024-09-13T14:53:08.825855+0000","last_became_peered":"2024-09-13T14:53:08.825855+0000","last_unstale":"2024-09-13T14:53:16.009791+0000","last_undegraded":"2024-09-13T14:53:08.770962+0000","last_fullsized":"2024-09-13T14:53:08.758698+0000","mapping_epoch":418,"log_start":"379'1213","ondisk_log_start":"379'1213","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'6020","last_scrub_stamp":"2024-09-13T14:51:14.900628+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:14.900628+0000","objects_scrubbed":3038,"log_size":7821,"log_dups_size":0,"ondisk_log_size":7821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":969,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84869120,"num_objects":1295,"num_object_clones":0,"num_object_copies":3885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":364,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3027,"num_write_kb":41088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":1295}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":543,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800860+0000","last_change":"2024-09-13T14:53:08.772753+0000","last_active":"2024-09-13T14:53:09.800860+0000","last_peered":"2024-09-13T14:53:09.800860+0000","last_clean":"2024-09-13T14:53:09.800860+0000","last_became_active":"2024-09-13T14:53:08.772283+0000","last_became_peered":"2024-09-13T14:53:08.772283+0000","last_unstale":"2024-09-13T14:53:09.800860+0000","last_undegraded":"2024-09-13T14:53:09.800860+0000","last_fullsized":"2024-09-13T14:53:09.800860+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:30.366156+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":945,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763023+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:53:09.763023+0000","last_peered":"2024-09-13T14:53:09.763023+0000","last_clean":"2024-09-13T14:53:09.763023+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:53:09.763023+0000","last_undegraded":"2024-09-13T14:53:09.763023+0000","last_fullsized":"2024-09-13T14:53:09.763023+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":685,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771458+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:53:11.771458+0000","last_peered":"2024-09-13T14:53:11.771458+0000","last_clean":"2024-09-13T14:53:11.771458+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:53:11.771458+0000","last_undegraded":"2024-09-13T14:53:11.771458+0000","last_fullsized":"2024-09-13T14:53:11.771458+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":387,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762245+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:53:09.762245+0000","last_peered":"2024-09-13T14:53:09.762245+0000","last_clean":"2024-09-13T14:53:09.762245+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:53:09.762245+0000","last_undegraded":"2024-09-13T14:53:09.762245+0000","last_fullsized":"2024-09-13T14:53:09.762245+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"421'12125","reported_seq":12253,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:17.066691+0000","last_change":"2024-09-13T14:53:09.825904+0000","last_active":"2024-09-13T14:53:17.066691+0000","last_peered":"2024-09-13T14:53:17.066691+0000","last_clean":"2024-09-13T14:53:17.066691+0000","last_became_active":"2024-09-13T14:53:09.800692+0000","last_became_peered":"2024-09-13T14:53:09.800692+0000","last_unstale":"2024-09-13T14:53:17.066691+0000","last_undegraded":"2024-09-13T14:53:17.066691+0000","last_fullsized":"2024-09-13T14:53:17.066691+0000","mapping_epoch":419,"log_start":"381'4402","ondisk_log_start":"381'4402","created":376,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":7723,"log_dups_size":0,"ondisk_log_size":7723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:23.781166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169934848,"num_objects":2593,"num_object_clones":0,"num_object_copies":7779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12125,"num_write_kb":470976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":538,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779441+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:53:11.779441+0000","last_peered":"2024-09-13T14:53:11.779441+0000","last_clean":"2024-09-13T14:53:11.779441+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:53:11.779441+0000","last_undegraded":"2024-09-13T14:53:11.779441+0000","last_fullsized":"2024-09-13T14:53:11.779441+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:52:04.371098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":833,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.821369+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:53:09.821369+0000","last_peered":"2024-09-13T14:53:09.821369+0000","last_clean":"2024-09-13T14:53:09.821369+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:53:09.821369+0000","last_undegraded":"2024-09-13T14:53:09.821369+0000","last_fullsized":"2024-09-13T14:53:09.821369+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848187+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:53:11.848187+0000","last_peered":"2024-09-13T14:53:11.848187+0000","last_clean":"2024-09-13T14:53:11.848187+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:53:11.848187+0000","last_undegraded":"2024-09-13T14:53:11.848187+0000","last_fullsized":"2024-09-13T14:53:11.848187+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848589+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:53:11.848589+0000","last_peered":"2024-09-13T14:53:11.848589+0000","last_clean":"2024-09-13T14:53:11.848589+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:53:11.848589+0000","last_undegraded":"2024-09-13T14:53:11.848589+0000","last_fullsized":"2024-09-13T14:53:11.848589+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"421'12036","reported_seq":12089,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.144193+0000","last_change":"2024-09-13T14:49:59.509683+0000","last_active":"2024-09-13T14:53:13.144193+0000","last_peered":"2024-09-13T14:53:13.144193+0000","last_clean":"2024-09-13T14:53:13.144193+0000","last_became_active":"2024-09-13T14:49:59.509390+0000","last_became_peered":"2024-09-13T14:49:59.509390+0000","last_unstale":"2024-09-13T14:53:13.144193+0000","last_undegraded":"2024-09-13T14:53:13.144193+0000","last_fullsized":"2024-09-13T14:53:13.144193+0000","mapping_epoch":376,"log_start":"381'5020","ondisk_log_start":"381'5020","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:08.503709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12036,"num_write_kb":472960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":572,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800675+0000","last_change":"2024-09-13T14:53:08.772285+0000","last_active":"2024-09-13T14:53:09.800675+0000","last_peered":"2024-09-13T14:53:09.800675+0000","last_clean":"2024-09-13T14:53:09.800675+0000","last_became_active":"2024-09-13T14:53:08.770613+0000","last_became_peered":"2024-09-13T14:53:08.770613+0000","last_unstale":"2024-09-13T14:53:09.800675+0000","last_undegraded":"2024-09-13T14:53:09.800675+0000","last_fullsized":"2024-09-13T14:53:09.800675+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:08.707450+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":711,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771437+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:53:11.771437+0000","last_peered":"2024-09-13T14:53:11.771437+0000","last_clean":"2024-09-13T14:53:11.771437+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:53:11.771437+0000","last_undegraded":"2024-09-13T14:53:11.771437+0000","last_fullsized":"2024-09-13T14:53:11.771437+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":671,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779280+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:53:11.779280+0000","last_peered":"2024-09-13T14:53:11.779280+0000","last_clean":"2024-09-13T14:53:11.779280+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:53:11.779280+0000","last_undegraded":"2024-09-13T14:53:11.779280+0000","last_fullsized":"2024-09-13T14:53:11.779280+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.799967+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:53:09.799967+0000","last_peered":"2024-09-13T14:53:09.799967+0000","last_clean":"2024-09-13T14:53:09.799967+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:53:09.799967+0000","last_undegraded":"2024-09-13T14:53:09.799967+0000","last_fullsized":"2024-09-13T14:53:09.799967+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"421'9091","reported_seq":9258,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:12.952660+0000","last_change":"2024-09-13T14:53:08.892889+0000","last_active":"2024-09-13T14:53:12.952660+0000","last_peered":"2024-09-13T14:53:12.952660+0000","last_clean":"2024-09-13T14:52:51.484266+0000","last_became_active":"2024-09-13T14:53:08.823426+0000","last_became_peered":"2024-09-13T14:53:08.823426+0000","last_unstale":"2024-09-13T14:53:12.952660+0000","last_undegraded":"2024-09-13T14:53:08.777484+0000","last_fullsized":"2024-09-13T14:53:08.758858+0000","mapping_epoch":418,"log_start":"379'828","ondisk_log_start":"379'828","created":376,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"384'6068","last_scrub_stamp":"2024-09-13T14:51:12.817531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:12.817531+0000","objects_scrubbed":3017,"log_size":8263,"log_dups_size":0,"ondisk_log_size":8263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:42.327124+0000","scrub_duration":783,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92274688,"num_objects":1408,"num_object_clones":0,"num_object_copies":4224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3033,"num_write_kb":45568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1408}],"blocked_by":[],"up_primary":0,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":259,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762665+0000","last_change":"2024-09-13T14:53:08.772053+0000","last_active":"2024-09-13T14:53:09.762665+0000","last_peered":"2024-09-13T14:53:09.762665+0000","last_clean":"2024-09-13T14:53:09.762665+0000","last_became_active":"2024-09-13T14:53:08.769631+0000","last_became_peered":"2024-09-13T14:53:08.769631+0000","last_unstale":"2024-09-13T14:53:09.762665+0000","last_undegraded":"2024-09-13T14:53:09.762665+0000","last_fullsized":"2024-09-13T14:53:09.762665+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:53.493391+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1036,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762552+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:53:09.762552+0000","last_peered":"2024-09-13T14:53:09.762552+0000","last_clean":"2024-09-13T14:53:09.762552+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:53:09.762552+0000","last_undegraded":"2024-09-13T14:53:09.762552+0000","last_fullsized":"2024-09-13T14:53:09.762552+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.764008+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:53:09.764008+0000","last_peered":"2024-09-13T14:53:09.764008+0000","last_clean":"2024-09-13T14:53:09.764008+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:53:09.764008+0000","last_undegraded":"2024-09-13T14:53:09.764008+0000","last_fullsized":"2024-09-13T14:53:09.764008+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":691,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779558+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:53:11.779558+0000","last_peered":"2024-09-13T14:53:11.779558+0000","last_clean":"2024-09-13T14:53:11.779558+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:53:11.779558+0000","last_undegraded":"2024-09-13T14:53:11.779558+0000","last_fullsized":"2024-09-13T14:53:11.779558+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"421'12274","reported_seq":12327,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.197230+0000","last_change":"2024-09-13T14:49:59.507705+0000","last_active":"2024-09-13T14:53:13.197230+0000","last_peered":"2024-09-13T14:53:13.197230+0000","last_clean":"2024-09-13T14:53:13.197230+0000","last_became_active":"2024-09-13T14:49:59.507374+0000","last_became_peered":"2024-09-13T14:49:59.507374+0000","last_unstale":"2024-09-13T14:53:13.197230+0000","last_undegraded":"2024-09-13T14:53:13.197230+0000","last_fullsized":"2024-09-13T14:53:13.197230+0000","mapping_epoch":376,"log_start":"381'5169","ondisk_log_start":"381'5169","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":7105,"log_dups_size":0,"ondisk_log_size":7105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:22.009750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12274,"num_write_kb":481088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":420,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762955+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:53:09.762955+0000","last_peered":"2024-09-13T14:53:09.762955+0000","last_clean":"2024-09-13T14:53:09.762955+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:53:09.762955+0000","last_undegraded":"2024-09-13T14:53:09.762955+0000","last_fullsized":"2024-09-13T14:53:09.762955+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":366,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763336+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:53:09.763336+0000","last_peered":"2024-09-13T14:53:09.763336+0000","last_clean":"2024-09-13T14:53:09.763336+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:53:09.763336+0000","last_undegraded":"2024-09-13T14:53:09.763336+0000","last_fullsized":"2024-09-13T14:53:09.763336+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1315,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:11.779738+0000","last_change":"2024-09-13T14:53:09.821938+0000","last_active":"2024-09-13T14:53:11.779738+0000","last_peered":"2024-09-13T14:53:11.779738+0000","last_clean":"2024-09-13T14:53:02.174586+0000","last_became_active":"2024-09-13T14:53:09.821475+0000","last_became_peered":"2024-09-13T14:53:09.821475+0000","last_unstale":"2024-09-13T14:53:11.779738+0000","last_undegraded":"2024-09-13T14:53:09.766813+0000","last_fullsized":"2024-09-13T14:53:09.765807+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":672,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779494+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:53:11.779494+0000","last_peered":"2024-09-13T14:53:11.779494+0000","last_clean":"2024-09-13T14:53:11.779494+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:53:11.779494+0000","last_undegraded":"2024-09-13T14:53:11.779494+0000","last_fullsized":"2024-09-13T14:53:11.779494+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"421'12365","reported_seq":12418,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.119727+0000","last_change":"2024-09-13T14:49:59.754749+0000","last_active":"2024-09-13T14:53:13.119727+0000","last_peered":"2024-09-13T14:53:13.119727+0000","last_clean":"2024-09-13T14:53:13.119727+0000","last_became_active":"2024-09-13T14:49:59.754398+0000","last_became_peered":"2024-09-13T14:49:59.754398+0000","last_unstale":"2024-09-13T14:53:13.119727+0000","last_undegraded":"2024-09-13T14:53:13.119727+0000","last_fullsized":"2024-09-13T14:53:13.119727+0000","mapping_epoch":376,"log_start":"393'6585","ondisk_log_start":"393'6585","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:08.387473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188678144,"num_objects":2879,"num_object_clones":0,"num_object_copies":8637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12365,"num_write_kb":487808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":440,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762215+0000","last_change":"2024-09-13T14:53:08.775380+0000","last_active":"2024-09-13T14:53:09.762215+0000","last_peered":"2024-09-13T14:53:09.762215+0000","last_clean":"2024-09-13T14:53:09.762215+0000","last_became_active":"2024-09-13T14:53:08.775030+0000","last_became_peered":"2024-09-13T14:53:08.775030+0000","last_unstale":"2024-09-13T14:53:09.762215+0000","last_undegraded":"2024-09-13T14:53:09.762215+0000","last_fullsized":"2024-09-13T14:53:09.762215+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:57.680954+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":681,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779609+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:53:11.779609+0000","last_peered":"2024-09-13T14:53:11.779609+0000","last_clean":"2024-09-13T14:53:11.779609+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:53:11.779609+0000","last_undegraded":"2024-09-13T14:53:11.779609+0000","last_fullsized":"2024-09-13T14:53:11.779609+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":837,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.831911+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:53:09.831911+0000","last_peered":"2024-09-13T14:53:09.831911+0000","last_clean":"2024-09-13T14:53:09.831911+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:53:09.831911+0000","last_undegraded":"2024-09-13T14:53:09.831911+0000","last_fullsized":"2024-09-13T14:53:09.831911+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848491+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:53:11.848491+0000","last_peered":"2024-09-13T14:53:11.848491+0000","last_clean":"2024-09-13T14:53:11.848491+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:53:11.848491+0000","last_undegraded":"2024-09-13T14:53:11.848491+0000","last_fullsized":"2024-09-13T14:53:11.848491+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"421'12117","reported_seq":12171,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.095303+0000","last_change":"2024-09-13T14:49:59.511444+0000","last_active":"2024-09-13T14:53:13.095303+0000","last_peered":"2024-09-13T14:53:13.095303+0000","last_clean":"2024-09-13T14:53:13.095303+0000","last_became_active":"2024-09-13T14:49:59.511154+0000","last_became_peered":"2024-09-13T14:49:59.511154+0000","last_unstale":"2024-09-13T14:53:13.095303+0000","last_undegraded":"2024-09-13T14:53:13.095303+0000","last_fullsized":"2024-09-13T14:53:13.095303+0000","mapping_epoch":376,"log_start":"389'6368","ondisk_log_start":"389'6368","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:18.382270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183762968,"num_objects":2805,"num_object_clones":0,"num_object_copies":8415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2805,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12117,"num_write_kb":477441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.818149+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:53:09.818149+0000","last_peered":"2024-09-13T14:53:09.818149+0000","last_clean":"2024-09-13T14:53:09.818149+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:53:09.818149+0000","last_undegraded":"2024-09-13T14:53:09.818149+0000","last_fullsized":"2024-09-13T14:53:09.818149+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":739,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771893+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:53:11.771893+0000","last_peered":"2024-09-13T14:53:11.771893+0000","last_clean":"2024-09-13T14:53:11.771893+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:53:11.771893+0000","last_undegraded":"2024-09-13T14:53:11.771893+0000","last_fullsized":"2024-09-13T14:53:11.771893+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1057,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.780192+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:53:11.780192+0000","last_peered":"2024-09-13T14:53:11.780192+0000","last_clean":"2024-09-13T14:53:11.780192+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:53:11.780192+0000","last_undegraded":"2024-09-13T14:53:11.780192+0000","last_fullsized":"2024-09-13T14:53:11.780192+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":729,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771562+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:53:11.771562+0000","last_peered":"2024-09-13T14:53:11.771562+0000","last_clean":"2024-09-13T14:53:11.771562+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:53:11.771562+0000","last_undegraded":"2024-09-13T14:53:11.771562+0000","last_fullsized":"2024-09-13T14:53:11.771562+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"421'12225","reported_seq":12337,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:16.140359+0000","last_change":"2024-09-13T14:49:59.755421+0000","last_active":"2024-09-13T14:53:16.140359+0000","last_peered":"2024-09-13T14:53:16.140359+0000","last_clean":"2024-09-13T14:53:16.140359+0000","last_became_active":"2024-09-13T14:49:59.755094+0000","last_became_peered":"2024-09-13T14:49:59.755094+0000","last_unstale":"2024-09-13T14:53:16.140359+0000","last_undegraded":"2024-09-13T14:53:16.140359+0000","last_fullsized":"2024-09-13T14:53:16.140359+0000","mapping_epoch":376,"log_start":"393'6757","ondisk_log_start":"393'6757","created":376,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:37:22.702709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171638784,"num_objects":2619,"num_object_clones":0,"num_object_copies":7857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12225,"num_write_kb":475008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":569,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771477+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:53:11.771477+0000","last_peered":"2024-09-13T14:53:11.771477+0000","last_clean":"2024-09-13T14:53:11.771477+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:53:11.771477+0000","last_undegraded":"2024-09-13T14:53:11.771477+0000","last_fullsized":"2024-09-13T14:53:11.771477+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:21.144311+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":20,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848695+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:53:11.848695+0000","last_peered":"2024-09-13T14:53:11.848695+0000","last_clean":"2024-09-13T14:53:11.848695+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:53:11.848695+0000","last_undegraded":"2024-09-13T14:53:11.848695+0000","last_fullsized":"2024-09-13T14:53:11.848695+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":839,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763031+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:53:09.763031+0000","last_peered":"2024-09-13T14:53:09.763031+0000","last_clean":"2024-09-13T14:53:09.763031+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:53:09.763031+0000","last_undegraded":"2024-09-13T14:53:09.763031+0000","last_fullsized":"2024-09-13T14:53:09.763031+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763808+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:53:09.763808+0000","last_peered":"2024-09-13T14:53:09.763808+0000","last_clean":"2024-09-13T14:53:09.763808+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:53:09.763808+0000","last_undegraded":"2024-09-13T14:53:09.763808+0000","last_fullsized":"2024-09-13T14:53:09.763808+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.764070+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:53:09.764070+0000","last_peered":"2024-09-13T14:53:09.764070+0000","last_clean":"2024-09-13T14:53:09.764070+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:53:09.764070+0000","last_undegraded":"2024-09-13T14:53:09.764070+0000","last_fullsized":"2024-09-13T14:53:09.764070+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":130,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.764066+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:53:09.764066+0000","last_peered":"2024-09-13T14:53:09.764066+0000","last_clean":"2024-09-13T14:53:09.764066+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:53:09.764066+0000","last_undegraded":"2024-09-13T14:53:09.764066+0000","last_fullsized":"2024-09-13T14:53:09.764066+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":668,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779394+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:53:11.779394+0000","last_peered":"2024-09-13T14:53:11.779394+0000","last_clean":"2024-09-13T14:53:11.779394+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:53:11.779394+0000","last_undegraded":"2024-09-13T14:53:11.779394+0000","last_fullsized":"2024-09-13T14:53:11.779394+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":420,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.823934+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:53:09.823934+0000","last_peered":"2024-09-13T14:53:09.823934+0000","last_clean":"2024-09-13T14:53:09.823934+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:53:09.823934+0000","last_undegraded":"2024-09-13T14:53:09.823934+0000","last_fullsized":"2024-09-13T14:53:09.823934+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":386,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763139+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:53:09.763139+0000","last_peered":"2024-09-13T14:53:09.763139+0000","last_clean":"2024-09-13T14:53:09.763139+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:53:09.763139+0000","last_undegraded":"2024-09-13T14:53:09.763139+0000","last_fullsized":"2024-09-13T14:53:09.763139+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":673,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771044+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:53:11.771044+0000","last_peered":"2024-09-13T14:53:11.771044+0000","last_clean":"2024-09-13T14:53:11.771044+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:53:11.771044+0000","last_undegraded":"2024-09-13T14:53:11.771044+0000","last_fullsized":"2024-09-13T14:53:11.771044+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.849134+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:53:11.849134+0000","last_peered":"2024-09-13T14:53:11.849134+0000","last_clean":"2024-09-13T14:53:11.849134+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:53:11.849134+0000","last_undegraded":"2024-09-13T14:53:11.849134+0000","last_fullsized":"2024-09-13T14:53:11.849134+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":729,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771357+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:53:11.771357+0000","last_peered":"2024-09-13T14:53:11.771357+0000","last_clean":"2024-09-13T14:53:11.771357+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:53:11.771357+0000","last_undegraded":"2024-09-13T14:53:11.771357+0000","last_fullsized":"2024-09-13T14:53:11.771357+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.12","version":"421'9252","reported_seq":19167,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:13.131803+0000","last_change":"2024-09-13T14:53:08.856844+0000","last_active":"2024-09-13T14:53:13.131803+0000","last_peered":"2024-09-13T14:53:13.131803+0000","last_clean":"2024-09-13T14:51:20.220249+0000","last_became_active":"2024-09-13T14:53:08.831561+0000","last_became_peered":"2024-09-13T14:53:08.831561+0000","last_unstale":"2024-09-13T14:53:13.131803+0000","last_undegraded":"2024-09-13T14:53:08.776031+0000","last_fullsized":"2024-09-13T14:53:08.759567+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":9252,"log_dups_size":0,"ondisk_log_size":9252,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92274688,"num_objects":1408,"num_object_clones":0,"num_object_copies":4224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":411,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6226,"num_write_kb":244288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3261,"num_bytes_recovered":213712896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,5","objects":1408}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":418,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763006+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:53:09.763006+0000","last_peered":"2024-09-13T14:53:09.763006+0000","last_clean":"2024-09-13T14:53:09.763006+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:53:09.763006+0000","last_undegraded":"2024-09-13T14:53:09.763006+0000","last_fullsized":"2024-09-13T14:53:09.763006+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":761,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771678+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:53:11.771678+0000","last_peered":"2024-09-13T14:53:11.771678+0000","last_clean":"2024-09-13T14:53:11.771678+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:53:11.771678+0000","last_undegraded":"2024-09-13T14:53:11.771678+0000","last_fullsized":"2024-09-13T14:53:11.771678+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"421'11000","reported_seq":20673,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.102102+0000","last_change":"2024-09-13T14:52:20.753132+0000","last_active":"2024-09-13T14:53:13.102102+0000","last_peered":"2024-09-13T14:53:13.102102+0000","last_clean":"2024-09-13T14:53:13.102102+0000","last_became_active":"2024-09-13T14:52:20.740728+0000","last_became_peered":"2024-09-13T14:52:20.740728+0000","last_unstale":"2024-09-13T14:53:13.102102+0000","last_undegraded":"2024-09-13T14:53:13.102102+0000","last_fullsized":"2024-09-13T14:53:13.102102+0000","mapping_epoch":397,"log_start":"381'4612","ondisk_log_start":"381'4612","created":383,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":2773,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175232,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":240960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3163,"num_bytes_recovered":207290368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848862+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:53:11.848862+0000","last_peered":"2024-09-13T14:53:11.848862+0000","last_clean":"2024-09-13T14:53:11.848862+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:53:11.848862+0000","last_undegraded":"2024-09-13T14:53:11.848862+0000","last_fullsized":"2024-09-13T14:53:11.848862+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":693,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779198+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:53:11.779198+0000","last_peered":"2024-09-13T14:53:11.779198+0000","last_clean":"2024-09-13T14:53:11.779198+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:53:11.779198+0000","last_undegraded":"2024-09-13T14:53:11.779198+0000","last_fullsized":"2024-09-13T14:53:11.779198+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.10","version":"421'8967","reported_seq":22914,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:12.824254+0000","last_change":"2024-09-13T14:53:08.878205+0000","last_active":"2024-09-13T14:53:12.824254+0000","last_peered":"2024-09-13T14:53:12.824254+0000","last_clean":"2024-09-13T14:52:51.773580+0000","last_became_active":"2024-09-13T14:53:08.824238+0000","last_became_peered":"2024-09-13T14:53:08.824238+0000","last_unstale":"2024-09-13T14:53:12.824254+0000","last_undegraded":"2024-09-13T14:53:08.878010+0000","last_fullsized":"2024-09-13T14:53:08.764024+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":383,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"384'5989","last_scrub_stamp":"2024-09-13T14:51:14.700148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:14.700148+0000","objects_scrubbed":3045,"log_size":8967,"log_dups_size":0,"ondisk_log_size":8967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:00.558573+0000","scrub_duration":958,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93061120,"num_objects":1420,"num_object_clones":0,"num_object_copies":4260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2989,"num_write_kb":43648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3809,"num_bytes_recovered":249561088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,7","objects":1420}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":742,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771584+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:53:11.771584+0000","last_peered":"2024-09-13T14:53:11.771584+0000","last_clean":"2024-09-13T14:53:11.771584+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:53:11.771584+0000","last_undegraded":"2024-09-13T14:53:11.771584+0000","last_fullsized":"2024-09-13T14:53:11.771584+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":119,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763769+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:53:09.763769+0000","last_peered":"2024-09-13T14:53:09.763769+0000","last_clean":"2024-09-13T14:53:09.763769+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:53:09.763769+0000","last_undegraded":"2024-09-13T14:53:09.763769+0000","last_fullsized":"2024-09-13T14:53:09.763769+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"421'9085","reported_seq":9199,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:13.072345+0000","last_change":"2024-09-13T14:51:18.887707+0000","last_active":"2024-09-13T14:53:13.072345+0000","last_peered":"2024-09-13T14:53:13.072345+0000","last_clean":"2024-09-13T14:53:13.072345+0000","last_became_active":"2024-09-13T14:51:18.883159+0000","last_became_peered":"2024-09-13T14:51:18.883159+0000","last_unstale":"2024-09-13T14:53:13.072345+0000","last_undegraded":"2024-09-13T14:53:13.072345+0000","last_fullsized":"2024-09-13T14:53:13.072345+0000","mapping_epoch":387,"log_start":"379'811","ondisk_log_start":"379'811","created":383,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":2899,"log_size":8274,"log_dups_size":0,"ondisk_log_size":8274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88113152,"num_objects":1345,"num_object_clones":0,"num_object_copies":4035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6085,"num_write_kb":237728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":639,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.779922+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:53:11.779922+0000","last_peered":"2024-09-13T14:53:11.779922+0000","last_clean":"2024-09-13T14:53:11.779922+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:53:11.779922+0000","last_undegraded":"2024-09-13T14:53:11.779922+0000","last_fullsized":"2024-09-13T14:53:11.779922+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":16,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.848595+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:53:11.848595+0000","last_peered":"2024-09-13T14:53:11.848595+0000","last_clean":"2024-09-13T14:53:11.848595+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:53:11.848595+0000","last_undegraded":"2024-09-13T14:53:11.848595+0000","last_fullsized":"2024-09-13T14:53:11.848595+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.16","version":"421'11447","reported_seq":15401,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:12.800721+0000","last_change":"2024-09-13T14:52:54.785486+0000","last_active":"2024-09-13T14:53:12.800721+0000","last_peered":"2024-09-13T14:53:12.800721+0000","last_clean":"2024-09-13T14:53:12.800721+0000","last_became_active":"2024-09-13T14:52:54.762221+0000","last_became_peered":"2024-09-13T14:52:54.762221+0000","last_unstale":"2024-09-13T14:53:12.800721+0000","last_undegraded":"2024-09-13T14:53:12.800721+0000","last_fullsized":"2024-09-13T14:53:12.800721+0000","mapping_epoch":414,"log_start":"380'3742","ondisk_log_start":"380'3742","created":383,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"404'10612","last_scrub_stamp":"2024-09-13T14:52:35.486546+0000","last_deep_scrub":"404'10612","last_deep_scrub_stamp":"2024-09-13T14:52:35.486546+0000","last_clean_scrub_stamp":"2024-09-13T14:52:35.486546+0000","objects_scrubbed":2381,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92635136,"num_objects":1414,"num_object_clones":0,"num_object_copies":4242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6094,"num_write_kb":240224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1888,"num_bytes_recovered":123731968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763982+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:53:09.763982+0000","last_peered":"2024-09-13T14:53:09.763982+0000","last_clean":"2024-09-13T14:53:09.763982+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:53:09.763982+0000","last_undegraded":"2024-09-13T14:53:09.763982+0000","last_fullsized":"2024-09-13T14:53:09.763982+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":386,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.762807+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:53:09.762807+0000","last_peered":"2024-09-13T14:53:09.762807+0000","last_clean":"2024-09-13T14:53:09.762807+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:53:09.762807+0000","last_undegraded":"2024-09-13T14:53:09.762807+0000","last_fullsized":"2024-09-13T14:53:09.762807+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.17","version":"421'8826","reported_seq":11302,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:53:13.069650+0000","last_change":"2024-09-13T14:53:08.934352+0000","last_active":"2024-09-13T14:53:13.069650+0000","last_peered":"2024-09-13T14:53:13.069650+0000","last_clean":"2024-09-13T14:52:36.944870+0000","last_became_active":"2024-09-13T14:53:08.879006+0000","last_became_peered":"2024-09-13T14:53:08.879006+0000","last_unstale":"2024-09-13T14:53:13.069650+0000","last_undegraded":"2024-09-13T14:53:08.867290+0000","last_fullsized":"2024-09-13T14:53:08.764159+0000","mapping_epoch":418,"log_start":"379'416","ondisk_log_start":"379'416","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:49:58.492850+0000","objects_scrubbed":0,"log_size":8410,"log_dups_size":0,"ondisk_log_size":8410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96337920,"num_objects":1470,"num_object_clones":0,"num_object_copies":4410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5914,"num_write_kb":236288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":795,"num_bytes_recovered":52101120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"1,7","objects":66},{"shards":"3,5,7","objects":1404}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763688+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:53:09.763688+0000","last_peered":"2024-09-13T14:53:09.763688+0000","last_clean":"2024-09-13T14:53:09.763688+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:53:09.763688+0000","last_undegraded":"2024-09-13T14:53:09.763688+0000","last_fullsized":"2024-09-13T14:53:09.763688+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":352,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.800154+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:53:09.800154+0000","last_peered":"2024-09-13T14:53:09.800154+0000","last_clean":"2024-09-13T14:53:09.800154+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:53:09.800154+0000","last_undegraded":"2024-09-13T14:53:09.800154+0000","last_fullsized":"2024-09-13T14:53:09.800154+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.14","version":"421'9075","reported_seq":9231,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:15.427626+0000","last_change":"2024-09-13T14:51:15.723730+0000","last_active":"2024-09-13T14:53:15.427626+0000","last_peered":"2024-09-13T14:53:15.427626+0000","last_clean":"2024-09-13T14:53:15.427626+0000","last_became_active":"2024-09-13T14:51:10.831017+0000","last_became_peered":"2024-09-13T14:51:10.831017+0000","last_unstale":"2024-09-13T14:53:15.427626+0000","last_undegraded":"2024-09-13T14:53:15.427626+0000","last_fullsized":"2024-09-13T14:53:15.427626+0000","mapping_epoch":383,"log_start":"379'1518","ondisk_log_start":"379'1518","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"384'6143","last_scrub_stamp":"2024-09-13T14:51:15.723432+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.723432+0000","objects_scrubbed":3022,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:03.475824+0000","scrub_duration":949,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2940,"num_write_kb":38080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":753,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771544+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:53:11.771544+0000","last_peered":"2024-09-13T14:53:11.771544+0000","last_clean":"2024-09-13T14:53:11.771544+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:53:11.771544+0000","last_undegraded":"2024-09-13T14:53:11.771544+0000","last_fullsized":"2024-09-13T14:53:11.771544+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":726,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.771372+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:53:11.771372+0000","last_peered":"2024-09-13T14:53:11.771372+0000","last_clean":"2024-09-13T14:53:11.771372+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:53:11.771372+0000","last_undegraded":"2024-09-13T14:53:11.771372+0000","last_fullsized":"2024-09-13T14:53:11.771372+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.15","version":"421'8943","reported_seq":13551,"reported_epoch":421,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-13T14:53:13.128559+0000","last_change":"2024-09-13T14:53:08.898658+0000","last_active":"2024-09-13T14:53:13.128559+0000","last_peered":"2024-09-13T14:53:13.128559+0000","last_clean":"2024-09-13T14:52:33.669447+0000","last_became_active":"2024-09-13T14:52:36.031984+0000","last_became_peered":"2024-09-13T14:52:36.031984+0000","last_unstale":"2024-09-13T14:53:13.128559+0000","last_undegraded":"2024-09-13T14:53:13.128559+0000","last_fullsized":"2024-09-13T14:52:35.855805+0000","mapping_epoch":404,"log_start":"379'624","ondisk_log_start":"379'624","created":383,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"384'6000","last_scrub_stamp":"2024-09-13T14:51:15.011294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:49:58.492850+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.011294+0000","objects_scrubbed":3000,"log_size":8319,"log_dups_size":0,"ondisk_log_size":8319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:59.522624+0000","scrub_duration":915,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85262336,"num_objects":1301,"num_object_clones":0,"num_object_copies":3903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1298,"num_objects_unfound":0,"num_objects_dirty":1301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2953,"num_write_kb":40128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1487,"num_bytes_recovered":97452032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":649},{"shards":"1,7","objects":652}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2024-09-13T14:53:09.763603+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:53:09.763603+0000","last_peered":"2024-09-13T14:53:09.763603+0000","last_clean":"2024-09-13T14:53:09.763603+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:53:09.763603+0000","last_undegraded":"2024-09-13T14:53:09.763603+0000","last_fullsized":"2024-09-13T14:53:09.763603+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":22,"reported_epoch":421,"state":"active+clean","last_fresh":"2024-09-13T14:53:11.849058+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:53:11.849058+0000","last_peered":"2024-09-13T14:53:11.849058+0000","last_clean":"2024-09-13T14:53:11.849058+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:53:11.849058+0000","last_undegraded":"2024-09-13T14:53:11.849058+0000","last_fullsized":"2024-09-13T14:53:11.849058+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":2873753624,"num_objects":43852,"num_object_clones":0,"num_object_copies":131556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8651,"num_objects_misplaced":1298,"num_objects_unfound":0,"num_objects_dirty":43852,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146415,"num_write_kb":4619521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14407,"num_bytes_recovered":943980544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9816518656,"data_stored":9816506440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":197911,"ondisk_log_size":197911,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398005,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2621380,"kb_used_data":1729528,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91123772,"statfs":{"total":95995035648,"available":93310742528,"internally_reserved":0,"allocated":1771036672,"data_stored":1768785569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594233,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1603888,"kb_used_data":709348,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":92141264,"statfs":{"total":95995035648,"available":94352654336,"internally_reserved":0,"allocated":726372352,"data_stored":724141003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665687,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2276140,"kb_used_data":1943588,"kb_used_omap":9,"kb_used_meta":332534,"kb_avail":91469012,"statfs":{"total":95995035648,"available":93664268288,"internally_reserved":0,"allocated":1990234112,"data_stored":1987999094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":340515016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986689,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633724,"kb_used_data":733872,"kb_used_omap":1,"kb_used_meta":899838,"kb_avail":92111428,"statfs":{"total":95995035648,"available":94322102272,"internally_reserved":0,"allocated":751484928,"data_stored":749241923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921434456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697249,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2577320,"kb_used_data":1598816,"kb_used_omap":4,"kb_used_meta":978491,"kb_avail":91167832,"statfs":{"total":95995035648,"available":93355859968,"internally_reserved":0,"allocated":1637187584,"data_stored":1634956719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1001974859},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860776,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1270636,"kb_used_data":545380,"kb_used_omap":8,"kb_used_meta":725239,"kb_avail":92474516,"statfs":{"total":95995035648,"available":94693904384,"internally_reserved":0,"allocated":558469120,"data_stored":556234673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":742645580},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020447,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1953704,"kb_used_data":1423392,"kb_used_omap":9,"kb_used_meta":530166,"kb_avail":91791448,"statfs":{"total":95995035648,"available":93994442752,"internally_reserved":0,"allocated":1457553408,"data_stored":1455321944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":542890821},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187838,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1854820,"kb_used_data":1070044,"kb_used_omap":7,"kb_used_meta":784760,"kb_avail":91890332,"statfs":{"total":95995035648,"available":94095699968,"internally_reserved":0,"allocated":1095725056,"data_stored":1093497930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":803594571},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1077018624,"data_stored":1077018624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1432752128,"data_stored":1432748056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":544079872,"data_stored":544079872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1607729152,"data_stored":1607729152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":740950016,"data_stored":740950016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1954287616,"data_stored":1954283544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":718209024,"data_stored":718209024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1741492224,"data_stored":1741488152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:53:20.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:53:20.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.354 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:20.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:20.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:20.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:20.528 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:20.620 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:20.620 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:20.649 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:20.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:20.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:20.899 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:20.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.054 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:20 smithi017 ceph-mon[103001]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 8348/122190 objects degraded (6.832%); 928/122190 objects misplaced (0.759%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T14:53:21.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:20 smithi017 ceph-mon[107556]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 8348/122190 objects degraded (6.832%); 928/122190 objects misplaced (0.759%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T14:53:21.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:21.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:20 smithi189 ceph-mon[90992]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 8348/122190 objects degraded (6.832%); 928/122190 objects misplaced (0.759%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T14:53:21.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:21.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:21.442 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:21.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:21.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:21.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.805 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:53:21.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:21.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:22.029 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:22.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:22.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3980205825' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3980205825' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3980205825' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:22.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:22.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:22.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.567 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:22.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:22.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:22.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:22.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.162 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[103001]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 8348/118071 objects degraded (7.070%); 928/118071 objects misplaced (0.786%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[103001]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[107556]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 8348/118071 objects degraded (7.070%); 928/118071 objects misplaced (0.786%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:22 smithi017 ceph-mon[107556]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:53:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:22 smithi189 ceph-mon[90992]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 8348/118071 objects degraded (7.070%); 928/118071 objects misplaced (0.786%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:53:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:22 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:22 smithi189 ceph-mon[90992]: osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:53:23.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:23.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:23.413 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.451 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.729 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:23.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:23.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.119 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:24.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:24.189 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:23 smithi017 ceph-mon[103001]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:23 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 8348/118071 objects degraded (7.070%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:23 smithi017 ceph-mon[107556]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:23 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 8348/118071 objects degraded (7.070%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:24.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:23 smithi189 ceph-mon[90992]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:24.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:23 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 8348/118071 objects degraded (7.070%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:24.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:24.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:24.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:24.533 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:24.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:24.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:24.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.981 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:25.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:25.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:25.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:25.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:25.411 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:25.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:25.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:25.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:25.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:25.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:25.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:26.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:26.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:26.385 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:26.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:26.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:26.672 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:26 smithi017 ceph-mon[103001]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:26 smithi017 ceph-mon[107556]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:26.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:26 smithi189 ceph-mon[90992]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 8050/106932 objects degraded (7.528%); 528/106932 objects misplaced (0.494%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:26.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:26.801 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:26.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:26.904 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.940 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:26.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:27.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:27.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:27.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:27.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:27.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:27.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:27.580 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:53:27.580 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:53:27.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:208509 9.12 e422/418 9:4b0901ba:::benchmark_data_smithi017_339564_object89036:head v 422'9687, mlcod=410'8743)", 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:27.574067+0000", 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0048385809999999998, 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048667069999999996, 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574067+0000", 2024-09-13T14:53:27.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574067+0000", 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574068+0000", 2024-09-13T14:53:27.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0109999999999999e-06 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574076+0000", 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1149999999999994e-06 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574094+0000", 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7125e-05 2024-09-13T14:53:27.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574102+0000", 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2749999999999995e-06 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574928+0000", 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082591999999999995 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:27.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.574950+0000", 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1909e-05 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:208513 9.19 e422/418 9:9e153962:::benchmark_data_smithi017_339564_object89040:head v 422'9604, mlcod=411'8633)", 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:27.578127+0000", 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00077923300000000005, 2024-09-13T14:53:27.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085661299999999999, 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578127+0000", 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:27.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578127+0000", 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578128+0000", 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.251e-06 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578135+0000", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.396e-06 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578147+0000", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1836e-05 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578155+0000", 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8129999999999996e-06 2024-09-13T14:53:27.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578254+0000", 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8966000000000001e-05 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:27.578277+0000", 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3339000000000001e-05 2024-09-13T14:53:27.589 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2024-09-13T14:53:27.590 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:53:27.595 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:27] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:53:27.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:27.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:27.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:27.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:27.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:27.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:28.032 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:28.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:28.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:28.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:28.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:28.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:28.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[103001]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 8050/101697 objects degraded (7.916%); 528/101697 objects misplaced (0.519%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:28 smithi017 ceph-mon[107556]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 8050/101697 objects degraded (7.916%); 528/101697 objects misplaced (0.519%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:28.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:28.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:53:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:28 smithi189 ceph-mon[90992]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 8050/101697 objects degraded (7.916%); 528/101697 objects misplaced (0.519%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:53:28.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:28.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:28.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:53:28.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:29.139 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:29 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 8050/101697 objects degraded (7.916%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:29.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:29.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:29.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:29.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:29.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:29 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 8050/101697 objects degraded (7.916%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:29.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:29.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:29 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 8050/101697 objects degraded (7.916%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:29.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:29.627 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:29.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.759 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:29.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:29.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:30.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:30.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:30.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:30.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:30 smithi017 ceph-mon[103001]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 7762/92124 objects degraded (8.426%); 56/92124 objects misplaced (0.061%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:30.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:30 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1919056449' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:30.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:30 smithi017 ceph-mon[107556]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 7762/92124 objects degraded (8.426%); 56/92124 objects misplaced (0.061%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:30.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:30 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1919056449' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:30.406 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:30.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:30 smithi189 ceph-mon[90992]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 7762/92124 objects degraded (8.426%); 56/92124 objects misplaced (0.061%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:30.407 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:30 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1919056449' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:30.420 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:53:30.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:53:30.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:53:30.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:53:30.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:53:30.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:53:30.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:53:30.496 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:30.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:30.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:30.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:31.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:31.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:31.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:31.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:31.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:53:31.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.684 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:31.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:31.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:31.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:32.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:32.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:32.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:32.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.411 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:32.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:32.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:32.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:32.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:32.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:32 smithi017 ceph-mon[103001]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7762/87840 objects degraded (8.837%); 56/87840 objects misplaced (0.064%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:32.705 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:32 smithi017 ceph-mon[107556]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7762/87840 objects degraded (8.837%); 56/87840 objects misplaced (0.064%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:32 smithi189 ceph-mon[90992]: pgmap v1403: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7762/87840 objects degraded (8.837%); 56/87840 objects misplaced (0.064%); 4.6 MiB/s, 72 objects/s recovering 2024-09-13T14:53:32.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.818 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:32.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:32.864 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:32.891 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:32.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:32.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:32.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:33.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:33.082 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:33.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.214 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:33.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:33.351 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:33.453 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:33.458 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-13T14:53:33.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:33.545 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:53:33.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:53:33.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:33.724 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:53:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:33.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:33.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:33.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:34.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:34.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:34.333 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:34.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:34.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:34.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:34.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:34.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:34.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[103001]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 7468/77172 objects degraded (9.677%); 4.9 MiB/s, 96 objects/s recovering 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[103001]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 7468/77172 objects degraded (9.677%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[107556]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 7468/77172 objects degraded (9.677%); 4.9 MiB/s, 96 objects/s recovering 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[107556]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:53:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:34 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 7468/77172 objects degraded (9.677%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:34.751 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:34 smithi189 ceph-mon[90992]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 7468/77172 objects degraded (9.677%); 4.9 MiB/s, 96 objects/s recovering 2024-09-13T14:53:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1378610552' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:53:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:34 smithi189 ceph-mon[90992]: osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:53:34.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:34 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 7468/77172 objects degraded (9.677%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:34.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:34.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:34.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:35.067 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:35.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:35.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:35.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:35.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:35.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:35.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:35.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:35.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:35.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:35.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:35.892 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:35.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:36.315 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:36.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:36.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:36.553 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.589 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:36.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:36.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:36.710 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:36 smithi017 ceph-mon[103001]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 7468/77172 objects degraded (9.677%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:36.711 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:36 smithi017 ceph-mon[107556]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 7468/77172 objects degraded (9.677%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:36.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:36 smithi189 ceph-mon[90992]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 991 op/s; 7468/77172 objects degraded (9.677%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:36.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:36.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:37.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:37.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:37.018 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:37.018 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:37.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:37.284 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:37.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:37.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:37.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:37.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:37.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:37.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.703 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:37] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:53:37.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:37.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:37.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:37.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:38.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:38.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:38.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:53:38.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:38.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:38.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:38.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:38.510 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:38.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:38 smithi017 ceph-mon[103001]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7468/71787 objects degraded (10.403%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:38.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:38 smithi017 ceph-mon[107556]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7468/71787 objects degraded (10.403%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:38.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.577 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:38.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:38.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:38 smithi189 ceph-mon[90992]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7468/71787 objects degraded (10.403%); 3.5 MiB/s, 76 objects/s recovering 2024-09-13T14:53:38.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:38.946 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.975 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:39.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:39.480 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:53:39.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 7468/71787 objects degraded (10.403%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:39.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/922928065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 7468/71787 objects degraded (10.403%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:39.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/922928065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:39.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:39 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 7468/71787 objects degraded (10.403%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/922928065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:39.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:39.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:39.883 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:39.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:40.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:40.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:40.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:40.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:40.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:40.479 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:40.489 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.522 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:40.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:40.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:40 smithi189 ceph-mon[90992]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7140/61563 objects degraded (11.598%); 2.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:40 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:40 smithi189 ceph-mon[90992]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:53:40.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:40.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:40.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:40.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[103001]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7140/61563 objects degraded (11.598%); 2.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[103001]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:53:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[107556]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7140/61563 objects degraded (11.598%); 2.1 MiB/s, 83 objects/s recovering 2024-09-13T14:53:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:40 smithi017 ceph-mon[107556]: osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:53:40.988 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:53:40.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:53:41.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:41.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:41.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:41.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:41.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:41.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:41.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:41.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:41.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:41.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:41.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:41.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.971 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:41.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:42.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:42.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:42.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:42.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:42.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:42.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:42.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:42.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:42.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:42.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:42.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:42.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:42 smithi189 ceph-mon[90992]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7140/57234 objects degraded (12.475%); 1.3 MiB/s, 55 objects/s recovering 2024-09-13T14:53:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:42.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:42.894 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:42.927 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:42 smithi017 ceph-mon[103001]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7140/57234 objects degraded (12.475%); 1.3 MiB/s, 55 objects/s recovering 2024-09-13T14:53:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:42 smithi017 ceph-mon[107556]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7140/57234 objects degraded (12.475%); 1.3 MiB/s, 55 objects/s recovering 2024-09-13T14:53:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:43.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.196 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:43.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:43.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:43.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:43.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:43.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:43.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:43.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:44.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:44.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:44.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:44.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:53:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.216 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:44.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:44.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:44.293 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:53:44.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:44.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:44.341 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:53:44.342 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:44.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:44.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:53:44.374 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:44.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:53:44.409 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:44.446 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:53:44.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:44.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:44.478 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:53:44.479 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:44.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:44.509 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:53:44.510 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:44.548 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:53:44.548 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:44.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:44.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[103001]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 92 objects/s recovering 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6806/46182 objects degraded (14.737%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/586234094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[107556]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 92 objects/s recovering 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6806/46182 objects degraded (14.737%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/586234094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:44.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:44.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:44 smithi189 ceph-mon[90992]: pgmap v1411: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 92 objects/s recovering 2024-09-13T14:53:44.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:44 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6806/46182 objects degraded (14.737%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:44.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/586234094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:44.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:44.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:44.887 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:44.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:45.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:45.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:45.198 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:45.300 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:45.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:45.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:45.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:45.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:45.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:45.704 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:45.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:45.870 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:45.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:46.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:46.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:46.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:46.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:46.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:46.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:46.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:46.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:46.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:46.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:46.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:46.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:46 smithi189 ceph-mon[90992]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:46.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:46.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:46 smithi017 ceph-mon[107556]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:46.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:46 smithi017 ceph-mon[103001]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 965 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/46182 objects degraded (14.737%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:46.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:46.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:47.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:47.155 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:47.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:47.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:47.391 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.429 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:47.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:47.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:47.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:47.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:47.688 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:53:47.688 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:53:47.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95174 9.5 9:a0c24c07:::benchmark_data_smithi017_339564_object95173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.772060+0000", 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.76719299800001, 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28612333499999998, 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.689 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95174 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772060+0000", 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772060+0000", 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772061+0000", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0999999999999997e-07 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772085+0000", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4085999999999999e-05 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772098+0000", 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3302000000000001e-05 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.772105+0000", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.109e-06 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781857+0000", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097514249999999993 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.782176+0000", 2024-09-13T14:53:47.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031907300000000001 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.782341+0000", 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016579999999999999 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791596+0000", 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0092541900000000007 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791607+0000", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1285e-05 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058084+0000", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26647682 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058183+0000", 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9434999999999999e-05 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.694 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95186 9.5 9.b2e11f45 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.793900+0000", 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.745352354, 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26478613699999998, 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.695 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95186 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793900+0000", 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793900+0000", 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793902+0000", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4699999999999999e-06 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793932+0000", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0507000000000001e-05 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794039+0000", 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106432 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794048+0000", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3670000000000003e-06 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058678+0000", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26462974299999997 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058687+0000", 2024-09-13T14:53:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6179999999999995e-06 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95187 9.9 9.9e20ab39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.794073+0000", 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.74517964099999, 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20393315400000001, 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.699 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95187 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794073+0000", 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794073+0000", 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794074+0000", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7199999999999999e-07 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794168+0000", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4136999999999997e-05 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794177+0000", 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2299999999999997e-06 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794183+0000", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7590000000000003e-06 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.997990+0000", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.203807456 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998006+0000", 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5999999999999999e-05 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.702 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95190 9.5 9.b95c5435 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.798994+0000", 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.74025854199999, 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25972665499999997, 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.703 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95190 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.798994+0000", 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.798994+0000", 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.798996+0000", 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0810000000000001e-06 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.799027+0000", 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0738000000000003e-05 2024-09-13T14:53:47.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.799052+0000", 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5292000000000001e-05 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.799063+0000", 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0601000000000001e-05 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058713+0000", 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25965046400000003 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058721+0000", 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4789999999999999e-06 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95191 9.9 9.933c8679 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:53:47.707 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.800435+0000", 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.73881828500001, 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19818255700000001, 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95191 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800435+0000", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800435+0000", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800436+0000", 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8959999999999999e-06 2024-09-13T14:53:47.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800461+0000", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5045000000000002e-05 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800480+0000", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9004999999999998e-05 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800489+0000", 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5990000000000002e-06 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998607+0000", 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19811832700000001 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998617+0000", 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6849999999999996e-06 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95174 9.5 9:a0c24c07:::benchmark_data_smithi017_339564_object95173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.711 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919439+0000", 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.619813956, 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90076942000000004, 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95174 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919439+0000", 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919439+0000", 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919441+0000", 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4600000000000002e-06 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919467+0000", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5814000000000001e-05 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919493+0000", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5493000000000001e-05 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919503+0000", 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0451999999999999e-05 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058745+0000", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13924239299999999 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058754+0000", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000001e-06 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.820120+0000", 2024-09-13T14:53:47.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.76136627800000001 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.820208+0000", 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7869999999999997e-05 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95186 9.5 9:a2f8874d:::benchmark_data_smithi017_339564_object95185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919523+0000", 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.61972935599999, 2024-09-13T14:53:47.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91190129200000003, 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95186 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919523+0000", 2024-09-13T14:53:47.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919523+0000", 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919524+0000", 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5400000000000002e-07 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919547+0000", 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2725e-05 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919554+0000", 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.824e-06 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919560+0000", 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8660000000000002e-06 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058806+0000", 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.139246114 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058813+0000", 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2960000000000004e-06 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.820632+0000", 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.76181901299999999 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.820780+0000", 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014854500000000001 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.820919+0000", 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013835799999999999 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825357+0000", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044386190000000004 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825402+0000", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4400999999999997e-05 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.831329+0000", 2024-09-13T14:53:47.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059277360000000003 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.831387+0000", 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7537999999999997e-05 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.831425+0000", 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7802999999999997e-05 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95187 9.19 9:9cd50479:::benchmark_data_smithi017_339564_object95186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919569+0000", 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.61968424299999, 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.93900693499999999, 2024-09-13T14:53:47.724 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95187 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919569+0000", 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919569+0000", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919569+0000", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6699999999999999e-07 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919589+0000", 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0140999999999999e-05 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919596+0000", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9770000000000003e-06 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919602+0000", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5439999999999998e-06 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998644+0000", 2024-09-13T14:53:47.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.079042668999999996 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998654+0000", 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833690+0000", 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83503609999999995 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833812+0000", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122044 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833942+0000", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129889 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839095+0000", 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051533170000000001 2024-09-13T14:53:47.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.841159+0000", 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020637450000000001 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.858481+0000", 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017322567000000001 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.858545+0000", 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3715999999999998e-05 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.858576+0000", 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0380000000000001e-05 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95190 9.15 9:ac2a3a9d:::benchmark_data_smithi017_339564_object95189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.731 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919610+0000", 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.61964239299999, 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92014164399999998, 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95190 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919610+0000", 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919610+0000", 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919611+0000", 2024-09-13T14:53:47.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5699999999999998e-07 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919637+0000", 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6148000000000001e-05 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919644+0000", 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7719999999999997e-06 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919649+0000", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6210000000000001e-06 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058775+0000", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13912586099999999 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.058781+0000", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8200000000000002e-06 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.834302+0000", 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.775521032 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.834419+0000", 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117255 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.834543+0000", 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012341899999999999 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839603+0000", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050604669999999999 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839640+0000", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7219999999999999e-05 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839662+0000", 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1033e-05 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839713+0000", 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1224000000000003e-05 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839752+0000", 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9314999999999999e-05 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.738 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95191 9.19 9:9e613cc9:::benchmark_data_smithi017_339564_object95190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919657+0000", 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.619595309, 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94342347999999998, 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95191 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919657+0000", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919657+0000", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919658+0000", 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9100000000000004e-07 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919678+0000", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9624e-05 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919684+0000", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5039999999999999e-06 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919689+0000", 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9339999999999997e-06 2024-09-13T14:53:47.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998678+0000", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078988869000000003 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.998685+0000", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6100000000000002e-06 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.835541+0000", 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.83685618399999995 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.835653+0000", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112188 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.835769+0000", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116358 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839102+0000", 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033330130000000001 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.841246+0000", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021437489999999999 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.862974+0000", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021727843 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.863036+0000", 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1680999999999994e-05 2024-09-13T14:53:47.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.863081+0000", 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5432000000000001e-05 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95196 9.5 9:a0e0bfaa:::benchmark_data_smithi017_339564_object95195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.919696+0000", 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.61955706699999, 2024-09-13T14:53:47.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91411300600000001, 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95196 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919696+0000", 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919696+0000", 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919696+0000", 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2199999999999999e-07 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919749+0000", 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2389000000000003e-05 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919756+0000", 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3150000000000003e-06 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919761+0000", 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.361e-06 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.059189+0000", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13942751 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.059199+0000", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9720000000000004e-06 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.823230+0000", 2024-09-13T14:53:47.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.764031762 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.823369+0000", 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013875000000000001 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.823529+0000", 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160111 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.826403+0000", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028735549999999999 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.826414+0000", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1195999999999999e-05 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833525+0000", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007110618 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833760+0000", 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023552600000000001 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.833809+0000", 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8519000000000001e-05 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:160221 9.10 e398/386 9:0f2097e9:::benchmark_data_smithi017_339564_object40748:head v 398'7940, mlcod=381'5598)", 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:25.182566+0000", 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 81.356686933000006, 2024-09-13T14:53:47.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15250518199999999, 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182566+0000", 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182566+0000", 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182567+0000", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7599999999999996e-07 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182574+0000", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182586+0000", 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1279e-05 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182593+0000", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6320000000000008e-06 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182722+0000", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012926499999999999 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.182746+0000", 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3343999999999999e-05 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.335007+0000", 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.152261278 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.335071+0000", 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3958e-05 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:160229 9.10 e398/386 9:0fe3ff23:::benchmark_data_smithi017_339564_object40756:head v 398'7941, mlcod=381'5598)", 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:25.187912+0000", 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 81.351341165999997, 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15055500799999999, 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.187912+0000", 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.187912+0000", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.187913+0000", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170000000000001e-06 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.187921+0000", 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1629999999999996e-06 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.324778+0000", 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13685719099999999 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.324794+0000", 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.615e-05 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.325318+0000", 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052384600000000001 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.325344+0000", 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6109e-05 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.338178+0000", 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01283394 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:25.338467+0000", 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000288492 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:206403 9.b 9:d5151ca4:::benchmark_data_smithi017_339564_object86930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.376098+0000", 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.163154389999999, 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21738389899999999, 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.761 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 206403 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376098+0000", 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376098+0000", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376100+0000", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.553e-06 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376106+0000", 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1299999999999998e-06 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376119+0000", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2473000000000001e-05 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376126+0000", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0029999999999997e-06 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376212+0000", 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6210000000000003e-05 2024-09-13T14:53:47.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376330+0000", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118292 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376442+0000", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111635 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.384401+0000", 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007959186 2024-09-13T14:53:47.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385570+0000", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011692600000000001 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593328+0000", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20775743799999999 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593436+0000", 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010806000000000001 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593482+0000", 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6659000000000002e-05 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:206405 9.b 9:d392b8b2:::benchmark_data_smithi017_339564_object86932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.376150+0000", 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.163103280000001, 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21749079700000001, 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.767 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 206405 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376150+0000", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376150+0000", 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376150+0000", 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7700000000000005e-07 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376152+0000", 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.368e-06 2024-09-13T14:53:47.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376159+0000", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3500000000000002e-06 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.376164+0000", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9470000000000002e-06 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.380374+0000", 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042106890000000001 2024-09-13T14:53:47.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.380496+0000", 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121333 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.380615+0000", 2024-09-13T14:53:47.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118998 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.384588+0000", 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039732439999999999 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579820+0000", 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19523183999999999 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593526+0000", 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013706312999999999 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593604+0000", 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8194000000000004e-05 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593640+0000", 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6044000000000003e-05 2024-09-13T14:53:47.773 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:206381 9.17 e422/418 9:eafb7db7:::benchmark_data_smithi017_339564_object86908:head v 422'9194, mlcod=381'5324)", 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.383040+0000", 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.156212645, 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20104572400000001, 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383040+0000", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383040+0000", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383041+0000", 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0100000000000001e-06 2024-09-13T14:53:47.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383048+0000", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8889999999999996e-06 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383061+0000", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2865e-05 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383068+0000", 2024-09-13T14:53:47.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5360000000000004e-06 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383142+0000", 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4004000000000005e-05 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.383169+0000", 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6296999999999999e-05 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.584051+0000", 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20088207199999999 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.584086+0000", 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5051e-05 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:206405 9.b e422/419 ondisk, result = 0)", 2024-09-13T14:53:47.778 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.385377+0000", 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.153875587000002, 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.194453338, 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385377+0000", 2024-09-13T14:53:47.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385377+0000", 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385378+0000", 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7399999999999998e-07 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385381+0000", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8169999999999999e-06 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385388+0000", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0010000000000004e-06 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385392+0000", 2024-09-13T14:53:47.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8729999999999998e-06 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579796+0000", 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19440318300000001 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579812+0000", 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6484999999999999e-05 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579831+0000", 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8505e-05 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:206416 9.12 e422/418 9:4dd547ab:::benchmark_data_smithi017_339564_object86943:head v 422'9625, mlcod=410'8743)", 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.385706+0000", 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.153547284999998, 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.197436741, 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385706+0000", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385706+0000", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385706+0000", 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1999999999999999e-07 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.385712+0000", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1360000000000002e-06 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579799+0000", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19408684700000001 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579811+0000", 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1581e-05 2024-09-13T14:53:47.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579888+0000", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7266000000000002e-05 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579912+0000", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3575999999999999e-05 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.583091+0000", 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031794750000000002 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.583142+0000", 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1140000000000002e-05 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:206409 9.19 e422/418 9:99829fd5:::benchmark_data_smithi017_339564_object86936:head v 422'9524, mlcod=411'8633)", 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.386013+0000", 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.153239330000002, 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.205986112, 2024-09-13T14:53:47.787 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.386013+0000", 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.386013+0000", 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.386014+0000", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2099999999999996e-07 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.386020+0000", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4890000000000003e-06 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579791+0000", 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19377127 2024-09-13T14:53:47.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579805+0000", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4368e-05 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579876+0000", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0556999999999995e-05 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579907+0000", 2024-09-13T14:53:47.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1603000000000002e-05 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.591949+0000", 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012041385999999999 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.592000+0000", 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0818e-05 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.791 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:206432 9.b 9:d19906ca:::benchmark_data_smithi017_339564_object86959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:53:25.402959+0000", 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 21.136293538, 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19089898899999999, 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 206432 2024-09-13T14:53:47.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.402959+0000", 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.402959+0000", 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:53:47.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.402962+0000", 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3369999999999998e-06 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.402970+0000", 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8750000000000006e-06 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579804+0000", 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17683325699999999 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.579814+0000", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0689e-05 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.580342+0000", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052800500000000001 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.580724+0000", 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000381655 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.584626+0000", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039022750000000002 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593492+0000", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088654830000000004 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593678+0000", 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000185959 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593739+0000", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1050999999999995e-05 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593822+0000", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3444999999999998e-05 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:53:25.593858+0000", 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5957999999999997e-05 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.797 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.798 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.798 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:53:47.798 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:53:47.798 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:53:47.798 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:47] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-09-13T14:53:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:47.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:47.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:47.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:48.047 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:48.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:48.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:48.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:48.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:48.362 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:48.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:48.615 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:48.651 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:48 smithi017 ceph-mon[107556]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 852 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/40770 objects degraded (16.694%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:48 smithi017 ceph-mon[103001]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 852 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/40770 objects degraded (16.694%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:48.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:48.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:48.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.919 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:48 smithi189 ceph-mon[90992]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 852 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6806/40770 objects degraded (16.694%); 2.1 MiB/s, 91 objects/s recovering 2024-09-13T14:53:49.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:49.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:49.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:49.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.523 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:53:49.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:53:49.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:49.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:49.799 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:49.822 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:49.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:49.935 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:49 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 6806/40770 objects degraded (16.694%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:49.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:49 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 6806/40770 objects degraded (16.694%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:49.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:49 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 6806/40770 objects degraded (16.694%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:50.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:50.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:50.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:50.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:50.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:53:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:53:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:50.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:50.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:50.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[107556]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 641 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6489/30639 objects degraded (21.179%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:50.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:50.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[107556]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:53:50.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[103001]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 641 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6489/30639 objects degraded (21.179%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:50.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:50.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:50 smithi017 ceph-mon[103001]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:53:50.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:50 smithi189 ceph-mon[90992]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 641 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 6489/30639 objects degraded (21.179%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1587783083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:50 smithi189 ceph-mon[90992]: osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:53:51.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:51.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:51.174 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:51.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.310 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:51.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:51.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:51.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:51.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:51.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:53:52.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.170 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:52.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:52.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:52.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:52.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:52.641 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:52.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:52.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:52.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:52.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:52.895 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:52 smithi017 ceph-mon[107556]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 555 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6489/26496 objects degraded (24.490%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:52 smithi017 ceph-mon[103001]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 555 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6489/26496 objects degraded (24.490%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:52.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:52.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:52 smithi189 ceph-mon[90992]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 555 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 6489/26496 objects degraded (24.490%); 2.0 MiB/s, 94 objects/s recovering 2024-09-13T14:53:53.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:53.091 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:53.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:53.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:53.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:53.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:53.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:53.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.512 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:53.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:53.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.650 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:53.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:53.751 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:53.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:53.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:54.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:54.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:54.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:54.304 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:54.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:54.552 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.654 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:54.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[107556]: pgmap v1417: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5743/15888 objects degraded (36.147%); 2.0 MiB/s, 105 objects/s recovering 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[107556]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5743/15888 objects degraded (36.147%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[103001]: pgmap v1417: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5743/15888 objects degraded (36.147%); 2.0 MiB/s, 105 objects/s recovering 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[103001]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:53:54.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:54 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5743/15888 objects degraded (36.147%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:54.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:54.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:55.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:54 smithi189 ceph-mon[90992]: pgmap v1417: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 5743/15888 objects degraded (36.147%); 2.0 MiB/s, 105 objects/s recovering 2024-09-13T14:53:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:54 smithi189 ceph-mon[90992]: osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:53:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:54 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5743/15888 objects degraded (36.147%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:55.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.348 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:55.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:55.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:55.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:55.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:55.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:55 smithi017 ceph-mon[103001]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:53:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:55 smithi017 ceph-mon[107556]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:53:55.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:55.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:56.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:55 smithi189 ceph-mon[90992]: osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:53:56.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:56.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:56.187 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:56.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:56.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.334 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:56.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:56.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:56.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:56.690 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:56.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:56.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:56.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:56.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:56 smithi017 ceph-mon[103001]: pgmap v1420: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 5743/15888 objects degraded (36.147%); 1.7 MiB/s, 96 objects/s recovering 2024-09-13T14:53:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:56 smithi017 ceph-mon[103001]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:53:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:56 smithi017 ceph-mon[107556]: pgmap v1420: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 5743/15888 objects degraded (36.147%); 1.7 MiB/s, 96 objects/s recovering 2024-09-13T14:53:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:56 smithi017 ceph-mon[107556]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:53:56.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:56.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:56 smithi189 ceph-mon[90992]: pgmap v1420: 109 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 334 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 5743/15888 objects degraded (36.147%); 1.7 MiB/s, 96 objects/s recovering 2024-09-13T14:53:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:56 smithi189 ceph-mon[90992]: osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:53:57.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:57.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:57.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:57.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:57.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:57.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:57.465 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:57.506 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:57.571 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:53:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:53:57] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-13T14:53:57.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.682 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:57.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:57.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:57 smithi017 ceph-mon[103001]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:53:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:57 smithi017 ceph-mon[107556]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:53:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:57.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:57 smithi189 ceph-mon[90992]: osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:53:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:58.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:58.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.390 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:58.610 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 119470 objects 2024-09-13T14:53:58.610 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :135.786 2024-09-13T14:53:58.612 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:53:58.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:58.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:58.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:58 smithi017 ceph-mon[103001]: pgmap v1423: 109 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 240 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 5379/11388 objects degraded (47.234%); 0 B/s, 24 objects/s recovering 2024-09-13T14:53:58.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:58 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:58.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:58 smithi017 ceph-mon[107556]: pgmap v1423: 109 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 240 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 5379/11388 objects degraded (47.234%); 0 B/s, 24 objects/s recovering 2024-09-13T14:53:58.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:58 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:58 smithi189 ceph-mon[90992]: pgmap v1423: 109 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 240 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 5379/11388 objects degraded (47.234%); 0 B/s, 24 objects/s recovering 2024-09-13T14:53:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:58 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:59.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:59.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.144 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:59.259 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:53:59.260 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:53:59.260 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:53:59.260 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:53:59.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.391 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.413 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:59.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:53:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:59.640 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:59.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:59.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:53:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[103001]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 5379/11388 objects degraded (47.234%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[107556]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:53:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:53:59 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 5379/11388 objects degraded (47.234%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:54:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:59 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:59 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1094051538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:59 smithi189 ceph-mon[90992]: osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:54:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:53:59 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 5379/11388 objects degraded (47.234%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:54:00.011 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:54:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:00.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:00.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:00.248 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:54:00.248 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:00.256 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:54:00.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:00.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:54:00.276 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:00.286 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.296 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:54:00.296 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:00.313 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:54:00.313 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:00.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:54:00.330 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:00.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:54:00.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:00.365 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:54:00.365 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:00.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:00.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:00.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:00.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:00.638 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:00.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:00.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:00 smithi189 ceph-mon[90992]: pgmap v1425: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 51 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 4759/2295 objects degraded (207.364%); 1.7 MiB/s, 183 objects/s recovering 2024-09-13T14:54:00.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:00 smithi189 ceph-mon[90992]: osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:54:00.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/469154910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:00.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:00.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:00.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[103001]: pgmap v1425: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 51 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 4759/2295 objects degraded (207.364%); 1.7 MiB/s, 183 objects/s recovering 2024-09-13T14:54:00.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[103001]: osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:54:00.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/469154910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[107556]: pgmap v1425: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 51 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 4759/2295 objects degraded (207.364%); 1.7 MiB/s, 183 objects/s recovering 2024-09-13T14:54:00.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[107556]: osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:54:00.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/469154910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:01.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:01.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:01.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:01.261 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:01.282 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:01.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:01.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:01.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:01.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:01.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:01.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:01.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:01.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:01.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[103001]: 9.16 scrub starts 2024-09-13T14:54:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[103001]: 9.16 scrub ok 2024-09-13T14:54:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[103001]: osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:54:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[107556]: 9.16 scrub starts 2024-09-13T14:54:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[107556]: 9.16 scrub ok 2024-09-13T14:54:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:01 smithi017 ceph-mon[107556]: osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:54:01.997 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:01 smithi189 ceph-mon[90992]: 9.16 scrub starts 2024-09-13T14:54:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:01 smithi189 ceph-mon[90992]: 9.16 scrub ok 2024-09-13T14:54:02.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:01 smithi189 ceph-mon[90992]: osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:54:02.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:02.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:02.136 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:02.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:02.385 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.406 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.759 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:02.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:54:02.884 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:02.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:02 smithi017 ceph-mon[103001]: pgmap v1428: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4759/1656 objects degraded (287.379%); 1.7 MiB/s, 163 objects/s recovering 2024-09-13T14:54:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:02 smithi017 ceph-mon[103001]: osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:54:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:02 smithi017 ceph-mon[107556]: pgmap v1428: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4759/1656 objects degraded (287.379%); 1.7 MiB/s, 163 objects/s recovering 2024-09-13T14:54:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:02 smithi017 ceph-mon[107556]: osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:54:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:02 smithi189 ceph-mon[90992]: pgmap v1428: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4759/1656 objects degraded (287.379%); 1.7 MiB/s, 163 objects/s recovering 2024-09-13T14:54:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:02 smithi189 ceph-mon[90992]: osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:54:03.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:03.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:03.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:03.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:03.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.281 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:03.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:03.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:03.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:03.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:03.781 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.804 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:03 smithi017 ceph-mon[103001]: osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:54:03.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:03 smithi017 ceph-mon[103001]: osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:54:03.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:03 smithi017 ceph-mon[107556]: osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:54:03.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:03 smithi017 ceph-mon[107556]: osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:54:03.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:03.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:03.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:04.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:03 smithi189 ceph-mon[90992]: osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:54:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:03 smithi189 ceph-mon[90992]: osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:54:04.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:04.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:04.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:04.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:04.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:04.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.405 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:04.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:04.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:04.478 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-13T14:54:04.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-13T14:54:04.542 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:04.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.670 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:54:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:04.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:04.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:04.950 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[107556]: pgmap v1431: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 15 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1151/561 objects degraded (205.169%) 2024-09-13T14:54:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[107556]: Health check update: Degraded data redundancy: 1151/561 objects degraded (205.169%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3952217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:04.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[103001]: pgmap v1431: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 15 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1151/561 objects degraded (205.169%) 2024-09-13T14:54:04.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[103001]: Health check update: Degraded data redundancy: 1151/561 objects degraded (205.169%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:04.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3952217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:04.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:04 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:04 smithi189 ceph-mon[90992]: pgmap v1431: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 15 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1151/561 objects degraded (205.169%) 2024-09-13T14:54:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:04 smithi189 ceph-mon[90992]: Health check update: Degraded data redundancy: 1151/561 objects degraded (205.169%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3952217475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:04 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:54:05.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.079 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:05.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.181 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:05.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.428 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:05.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:05.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:05.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:05.680 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:05.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:05.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:05.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:05.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:05 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:54:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:05 smithi017 ceph-mon[103001]: osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:54:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:05 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:54:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:05 smithi017 ceph-mon[107556]: osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:54:05.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:05.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:05 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:05 smithi189 ceph-mon[90992]: osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:54:06.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:06.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:06.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:54:06.518 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:06.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:06.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:06.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:06.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:06.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[103001]: pgmap v1434: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 15 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 755/561 objects degraded (134.581%) 2024-09-13T14:54:06.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[103001]: 9.12 scrub starts 2024-09-13T14:54:06.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[103001]: 9.12 scrub ok 2024-09-13T14:54:06.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[107556]: pgmap v1434: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 15 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 755/561 objects degraded (134.581%) 2024-09-13T14:54:06.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[107556]: 9.12 scrub starts 2024-09-13T14:54:06.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:06 smithi017 ceph-mon[107556]: 9.12 scrub ok 2024-09-13T14:54:06.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:06.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:06.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:06 smithi189 ceph-mon[90992]: pgmap v1434: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 15 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 755/561 objects degraded (134.581%) 2024-09-13T14:54:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:06 smithi189 ceph-mon[90992]: 9.12 scrub starts 2024-09-13T14:54:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:06 smithi189 ceph-mon[90992]: 9.12 scrub ok 2024-09-13T14:54:07.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:07.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:07.184 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:07.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:07.312 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:07.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:07.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.471 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:07] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-13T14:54:07.566 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:07.672 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:07.672 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:07.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:07.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:07.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:07 smithi017 ceph-mon[103001]: osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:54:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:07 smithi017 ceph-mon[107556]: osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:54:07.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:08.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:07 smithi189 ceph-mon[90992]: osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:54:08.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:08.099 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:08.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:08.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:08.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:08.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:08.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:54:08.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:08.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:08.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:08.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:08.638 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:08.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:08.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:08.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:08.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[103001]: pgmap v1436: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 13 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2/489 objects degraded (0.409%) 2024-09-13T14:54:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[103001]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/561 objects degraded (134.581%), 3 pgs degraded) 2024-09-13T14:54:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[103001]: osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:54:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[107556]: pgmap v1436: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 13 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2/489 objects degraded (0.409%) 2024-09-13T14:54:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[107556]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/561 objects degraded (134.581%), 3 pgs degraded) 2024-09-13T14:54:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:08 smithi017 ceph-mon[107556]: osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:54:08.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:08.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:08 smithi189 ceph-mon[90992]: pgmap v1436: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 13 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 2/489 objects degraded (0.409%) 2024-09-13T14:54:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:08 smithi189 ceph-mon[90992]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 755/561 objects degraded (134.581%), 3 pgs degraded) 2024-09-13T14:54:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:08 smithi189 ceph-mon[90992]: osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:54:09.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:09.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:09.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:09.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.446 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:09.446 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:09.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:09.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:09.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:09.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:09.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:09.707 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:54:09.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:09.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:09.852 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:09.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:09.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[107556]: 9.11 scrub starts 2024-09-13T14:54:09.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[107556]: 9.11 scrub ok 2024-09-13T14:54:09.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2569709918' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[103001]: 9.11 scrub starts 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[103001]: 9.11 scrub ok 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2569709918' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:09 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:54:09.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:10.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:09 smithi189 ceph-mon[90992]: 9.11 scrub starts 2024-09-13T14:54:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:09 smithi189 ceph-mon[90992]: 9.11 scrub ok 2024-09-13T14:54:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2569709918' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:09 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-13T14:54:10.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.127 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:10.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:10.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:10.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:10.479 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:10.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:10.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:10.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:10.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:10.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: pgmap v1438: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[103001]: osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: pgmap v1438: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:54:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:10 smithi017 ceph-mon[107556]: osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:54:10.979 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:11.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: pgmap v1438: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:54:11.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-13T14:54:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:10 smithi189 ceph-mon[90992]: osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:54:11.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:11.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:11.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:11.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:11.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:11.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.378 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:11.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:11.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:11.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:11.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:11.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:11.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:11.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:11.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:11.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:11 smithi189 ceph-mon[90992]: osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:54:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:11 smithi189 ceph-mon[90992]: 9.13 deep-scrub starts 2024-09-13T14:54:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:11 smithi189 ceph-mon[90992]: 9.13 deep-scrub ok 2024-09-13T14:54:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:54:12.007 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[103001]: osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:54:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[103001]: 9.13 deep-scrub starts 2024-09-13T14:54:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[103001]: 9.13 deep-scrub ok 2024-09-13T14:54:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:54:12.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[107556]: osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:54:12.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[107556]: 9.13 deep-scrub starts 2024-09-13T14:54:12.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[107556]: 9.13 deep-scrub ok 2024-09-13T14:54:12.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:54:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.235 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.258 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:12.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:12.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:12.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:12.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: pgmap v1441: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:13.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: pgmap v1441: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:13.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: pgmap v1441: 109 pgs: 109 active+clean; 13 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:54:13.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:13.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:13.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:13.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:13.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:13.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.370 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:13.472 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:13.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:13.619 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:13.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:13.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:13.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:13.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:13 smithi017 ceph-mon[107556]: osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:54:13.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:13 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]} : dispatch 2024-09-13T14:54:13.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:13 smithi017 ceph-mon[103001]: osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:54:13.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:13 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]} : dispatch 2024-09-13T14:54:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:13.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.994 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:13 smithi189 ceph-mon[90992]: osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:54:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:13 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]} : dispatch 2024-09-13T14:54:14.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:14.118 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:14.220 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:14.243 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:14.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.594 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:14.617 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:14.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:14.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:14.799 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: pgmap v1444: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:14.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]}]': finished 2024-09-13T14:54:14.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:54:14.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: 9.17 deep-scrub starts 2024-09-13T14:54:14.801 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:54:14.801 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: 9.17 deep-scrub ok 2024-09-13T14:54:14.801 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:14 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:14.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.867 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:14.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: pgmap v1444: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:14.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]}]': finished 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: 9.17 deep-scrub starts 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: 9.17 deep-scrub ok 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: pgmap v1444: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.18", "id": [7, 2]}]': finished 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:54:14.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 429'10562 2024-09-13T14:54:14.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: 9.17 deep-scrub starts 2024-09-13T14:54:14.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:54:14.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: 9.17 deep-scrub ok 2024-09-13T14:54:14.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:14 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:14.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:14.990 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:15.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:15.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:15.264 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:54:15.265 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:15.275 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:54:15.275 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:15.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:54:15.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:15.310 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:54:15.310 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:15.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:54:15.326 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:15.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:15.343 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:54:15.344 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:15.359 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:54:15.360 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:15.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:15.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:15.382 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:54:15.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:15.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:15.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:15.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.622 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:15.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:15.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:15.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:15.848 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:15.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[103001]: 9.18 starting backfill to osd.2 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[103001]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:54:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/648099091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[107556]: 9.18 starting backfill to osd.2 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[107556]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:54:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/648099091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:15.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:15.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:16.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.117 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:16.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:15 smithi189 ceph-mon[90992]: 9.18 starting backfill to osd.2 from (0'0,0'0] MAX to 429'10431 2024-09-13T14:54:16.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:15 smithi189 ceph-mon[90992]: osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:54:16.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/648099091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:16.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:54:16.344 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:16.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.468 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.490 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:16.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:16.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:16.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:16.779 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:54:16.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[103001]: pgmap v1448: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[103001]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3896328481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[107556]: pgmap v1448: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[107556]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3896328481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:16.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:16 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:16.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:16.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:16.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:16.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.090 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:17.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:17.178 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:16 smithi189 ceph-mon[90992]: pgmap v1448: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:17.179 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:16 smithi189 ceph-mon[90992]: osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:54:17.179 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3896328481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.179 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:16 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:17.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:17.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:17] "GET /metrics HTTP/1.1" 200 36651 "" "Prometheus/2.51.0" 2024-09-13T14:54:17.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:17.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:17.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.790 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' removed 2024-09-13T14:54:17.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:17.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[103001]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1203971534' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[107556]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1203971534' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:17 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:17.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:17.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:17 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:18.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:17 smithi189 ceph-mon[90992]: osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:54:18.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1203971534' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:18.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:17 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:54:18.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:17 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:18.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:18.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:18.240 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:18.342 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:18.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:18.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:18.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:18.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.621 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:18.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:18.681 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:54:18.682 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:54:18.682 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-13T14:54:18.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-13T14:54:18.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:18.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:18.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:18.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[103001]: pgmap v1451: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[103001]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[107556]: pgmap v1451: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:54:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[107556]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:54:18.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:18.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:18 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:18.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:19.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:19.103 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35952.0:127568 8.4 8:2cc62b5e:::benchmark_data_smithi017_253885_object36227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 501.958799892, 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17335995600000001, 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.104 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35952", 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4228251543", 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 127568 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213751+0000", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213753+0000", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0940000000000002e-06 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213760+0000", 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3869999999999997e-06 2024-09-13T14:54:19.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213777+0000", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6974e-05 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213787+0000", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5759999999999995e-06 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.213845+0000", 2024-09-13T14:54:19.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7785000000000003e-05 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.214225+0000", 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038041600000000001 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220181+0000", 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059560539999999997 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220236+0000", 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5072999999999999e-05 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.220317+0000", 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0638999999999998e-05 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.224135+0000", 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038185010000000002 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.224973+0000", 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00083726000000000004 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:19.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226407+0000", 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001434664 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226447+0000", 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9643e-05 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.387111+0000", 2024-09-13T14:54:19.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16066389 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.35952.0:127568 8.4 e355/317 ondisk, result = 0)", 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 501.94620516200001, 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16077082700000001, 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226346+0000", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5400000000000002e-07 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226349+0000", 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.033e-06 2024-09-13T14:54:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226356+0000", 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5039999999999999e-06 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226359+0000", 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6990000000000001e-06 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226398+0000", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8414999999999997e-05 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.226403+0000", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.57e-06 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.387116+0000", 2024-09-13T14:54:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.160713152 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35952.0:127569 8.1 e355/303 8:847fb8fd:::benchmark_data_smithi017_253885_object36228:head v 355'7703, mlcod=355'7701)", 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 501.94551553500003, 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16228176999999999, 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227035+0000", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227036+0000", 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8700000000000005e-07 2024-09-13T14:54:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.227043+0000", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8519999999999998e-06 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385983+0000", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15894031 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.385993+0000", 2024-09-13T14:54:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8260000000000005e-06 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386049+0000", 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5847999999999998e-05 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.386073+0000", 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4431999999999999e-05 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389281+0000", 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032075879999999999 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:45:55.389317+0000", 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6127000000000002e-05 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95178 9.2 e382/376 9:450e6495:::benchmark_data_smithi017_339564_object95177:head v 382'6050, mlcod=382'6045)", 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.781939+0000", 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 187.39061197500001, 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23103721599999999, 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781939+0000", 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781939+0000", 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781940+0000", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1039999999999999e-06 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781964+0000", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3836999999999999e-05 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781982+0000", 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8719000000000002e-05 2024-09-13T14:54:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781991+0000", 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2849999999999995e-06 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.782069+0000", 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8036999999999998e-05 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.782091+0000", 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2648999999999999e-05 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.012976+0000", 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.230884585 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95182 9.0 e382/376 9:0107e64f:::benchmark_data_smithi017_339564_object95181:head v 382'5869, mlcod=382'5867)", 2024-09-13T14:54:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.791899+0000", 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 187.38065209800001, 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18893632199999999, 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791899+0000", 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791899+0000", 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791900+0000", 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.393e-06 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791927+0000", 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6886e-05 2024-09-13T14:54:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791947+0000", 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0236000000000001e-05 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.791957+0000", 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4080000000000004e-06 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792058+0000", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101468 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792077+0000", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9355e-05 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.980835+0000", 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18875757600000001 2024-09-13T14:54:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95181 9.2 e382/376 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head v 382'6051, mlcod=382'6047)", 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.792155+0000", 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 187.38039595500001, 2024-09-13T14:54:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22083045700000001, 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792155+0000", 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792155+0000", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792155+0000", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0100000000000004e-07 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792178+0000", 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2382e-05 2024-09-13T14:54:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792187+0000", 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1179999999999998e-06 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792193+0000", 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7320000000000003e-06 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792270+0000", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6797000000000003e-05 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792288+0000", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8362000000000001e-05 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.012985+0000", 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22069736500000001 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95183 9.0 e382/376 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head v 382'5870, mlcod=382'5867)", 2024-09-13T14:54:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.793578+0000", 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 187.37897226000001, 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.187266031, 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793578+0000", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793578+0000", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793580+0000", 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0550000000000001e-06 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793604+0000", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4037e-05 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793615+0000", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1463e-05 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793622+0000", 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.204e-06 2024-09-13T14:54:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795341+0000", 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001718977 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795362+0000", 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0574999999999999e-05 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.980845+0000", 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18548271999999999 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95184 9.2 e382/376 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head v 382'6052, mlcod=382'6047)", 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.795396+0000", 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 187.37715429100001, 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.217593184, 2024-09-13T14:54:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795396+0000", 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795396+0000", 2024-09-13T14:54:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795397+0000", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3799999999999996e-07 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795423+0000", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5613e-05 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795435+0000", 2024-09-13T14:54:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1739000000000001e-05 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795443+0000", 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0290000000000005e-06 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795771+0000", 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032787300000000001 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.795789+0000", 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8601e-05 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.012990+0000", 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.217200491 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:47007 9.16 e398/383 9:687ec269:::benchmark_data_smithi017_339564_object66187:head v 398'8136, mlcod=398'8114)", 2024-09-13T14:54:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.736190+0000", 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.436360744, 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081751484999999999, 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736190+0000", 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736190+0000", 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736191+0000", 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6700000000000003e-07 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736196+0000", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0540000000000002e-06 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736204+0000", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5769999999999994e-06 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736208+0000", 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0829999999999997e-06 2024-09-13T14:54:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736272+0000", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4053000000000005e-05 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.736283+0000", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.11e-05 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.817854+0000", 2024-09-13T14:54:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081570474000000004 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.817941+0000", 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7577000000000005e-05 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:47008 9.16 e398/383 9:68823a4a:::benchmark_data_smithi017_339564_object79800:head v 398'8137, mlcod=398'8114)", 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.737269+0000", 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.435281719, 2024-09-13T14:54:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.080727288999999994, 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.737269+0000", 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.737269+0000", 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.737270+0000", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7100000000000002e-07 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.737274+0000", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9419999999999997e-06 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.805795+0000", 2024-09-13T14:54:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.068521296999999995 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.805805+0000", 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8900000000000002e-06 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.807897+0000", 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002092001 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.807917+0000", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0543999999999998e-05 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.817944+0000", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010026927 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.817996+0000", 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2117000000000003e-05 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:47035 9.16 e398/383 9:68886c73:::benchmark_data_smithi017_339564_object100166:head v 398'8145, mlcod=398'8141)", 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.909080+0000", 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.26347085899999, 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25971301200000002, 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909080+0000", 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909080+0000", 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909081+0000", 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2810000000000001e-06 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909088+0000", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909102+0000", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3356000000000001e-05 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909109+0000", 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.25e-06 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909166+0000", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6765000000000003e-05 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.909186+0000", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0188999999999999e-05 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168752+0000", 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25956596100000001 2024-09-13T14:54:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168793+0000", 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0951000000000001e-05 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:47036 9.16 e398/383 9:6889503b:::benchmark_data_smithi017_339564_object57331:head v 398'8146, mlcod=398'8141)", 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.911906+0000", 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.26064495200001, 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26340954, 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.911906+0000", 2024-09-13T14:54:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.911906+0000", 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.911907+0000", 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0690000000000001e-06 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.911914+0000", 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3849999999999996e-06 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168728+0000", 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.256814229 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168743+0000", 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.471e-05 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168827+0000", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3463999999999999e-05 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168847+0000", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0154999999999999e-05 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.175276+0000", 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006428886 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.175315+0000", 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9641999999999998e-05 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:159234 9.0 e398/383 9:0446345c:::benchmark_data_smithi017_339564_object39761:head v 398'7865, mlcod=398'7863)", 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.937744+0000", 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.234806885, 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23985853100000001, 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.937744+0000", 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.937744+0000", 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.937745+0000", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2780000000000001e-06 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.937754+0000", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7809999999999994e-06 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168732+0000", 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23097762199999999 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168743+0000", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.11e-05 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168815+0000", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1965999999999995e-05 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.168838+0000", 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3079e-05 2024-09-13T14:54:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.177568+0000", 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087300139999999995 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.177602+0000", 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4690999999999997e-05 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:159239 9.16 e398/383 9:6f7bc7d4:::benchmark_data_smithi017_339564_object39766:head v 398'8171, mlcod=398'8143)", 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:24.169284+0000", 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.003266594, 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074715606000000004, 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169284+0000", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169284+0000", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169284+0000", 2024-09-13T14:54:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6500000000000001e-07 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169287+0000", 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3709999999999998e-06 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169291+0000", 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.809e-06 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169295+0000", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.529e-06 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.231078+0000", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.061783101 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.231107+0000", 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8835000000000001e-05 2024-09-13T14:54:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.243955+0000", 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01284832 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244000+0000", 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4475999999999999e-05 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:159269 9.6 e398/383 9:66d3e9cd:::benchmark_data_smithi017_339564_object39796:head v 398'7938, mlcod=398'7935)", 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:24.169300+0000", 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.003251115, 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074737709999999999, 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169300+0000", 2024-09-13T14:54:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169300+0000", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169300+0000", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4600000000000001e-07 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169307+0000", 2024-09-13T14:54:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9990000000000002e-06 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169312+0000", 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1170000000000003e-06 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169316+0000", 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8039999999999999e-06 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.231135+0000", 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.061818834000000003 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.231152+0000", 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7795000000000002e-05 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244007+0000", 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012854325 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244037+0000", 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0689999999999999e-05 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:159270 9.16 e398/383 9:6e59d64c:::benchmark_data_smithi017_339564_object39797:head v 398'8172, mlcod=398'8143)", 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:24.169321+0000", 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.003230094, 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074906913000000006, 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169321+0000", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169321+0000", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169321+0000", 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8300000000000001e-07 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169323+0000", 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4959999999999999e-06 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169328+0000", 2024-09-13T14:54:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8539999999999996e-06 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169332+0000", 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6500000000000002e-06 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.234459+0000", 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.065127132000000004 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.234482+0000", 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2694999999999999e-05 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244196+0000", 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097148449999999997 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244228+0000", 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1058e-05 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:159287 9.6 e398/383 9:66502539:::benchmark_data_smithi017_339564_object39814:head v 398'7939, mlcod=398'7935)", 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:24.169338+0000", 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 113.003212283, 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.074926662000000005, 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169338+0000", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169338+0000", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169339+0000", 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6899999999999999e-07 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169341+0000", 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5040000000000001e-06 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169346+0000", 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6569999999999996e-06 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.169349+0000", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6880000000000001e-06 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.236898+0000", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.067548078999999997 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.236916+0000", 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8308000000000002e-05 2024-09-13T14:54:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244234+0000", 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007317685 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.244265+0000", 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1572000000000002e-05 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:49529 9.16 e400/383 9:6c8e32be:::benchmark_data_smithi017_339564_object93677:head v 400'9535, mlcod=400'9533)", 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:30.575981+0000", 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 106.596569285, 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.070212986000000005, 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.575981+0000", 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.575981+0000", 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.575983+0000", 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.533e-06 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.575995+0000", 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2374e-05 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.576019+0000", 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3289999999999999e-05 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.576028+0000", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3640000000000005e-06 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.576139+0000", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111316 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.576160+0000", 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0322e-05 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.646147+0000", 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.069987087000000003 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:30.646194+0000", 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7700000000000001e-05 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.5.0:50454 9.16 e402/383 9:6e2817a0:::benchmark_data_smithi017_339564_object75464:head v 402'10041, mlcod=402'10013)", 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:32.967936+0000", 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.20461439899999, 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.075526619000000003, 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967936+0000", 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967936+0000", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967937+0000", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0979999999999999e-06 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967944+0000", 2024-09-13T14:54:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8549999999999996e-06 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967959+0000", 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5131e-05 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.967967+0000", 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3669999999999999e-06 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.022688+0000", 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.054720984 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.022714+0000", 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6444999999999998e-05 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.043369+0000", 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020655066999999999 2024-09-13T14:54:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.043463+0000", 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3671999999999994e-05 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:166446 9.16 e402/383 9:6c8c2a56:::benchmark_data_smithi017_339564_object46973:head v 402'10042, mlcod=402'10019)", 2024-09-13T14:54:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:32.973057+0000", 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.19949328200001, 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.070465027, 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973057+0000", 2024-09-13T14:54:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973057+0000", 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973058+0000", 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5899999999999996e-07 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973062+0000", 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1400000000000002e-06 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973073+0000", 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.076e-05 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:32.973078+0000", 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5589999999999996e-06 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.026032+0000", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.052953326000000002 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.026051+0000", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9114e-05 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.043467+0000", 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017416445999999999 2024-09-13T14:54:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:33.043522+0000", 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5123000000000001e-05 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:19.202 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:54:19.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:18 smithi189 ceph-mon[90992]: pgmap v1451: 109 pgs: 1 peering, 108 active+clean; 8.7 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:18 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:54:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:18 smithi189 ceph-mon[90992]: osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:54:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:18 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:19.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:19.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:19.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:19.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:19.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:19.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:54:19.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:19.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:19.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.084 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.111 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.213 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:54:20.239 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.286 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.286 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.305 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.336 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:20.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:20.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:20.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:20.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:20.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:20.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:20.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:20.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:20.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:20.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:20.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:21.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:20 smithi017 ceph-mon[103001]: pgmap v1453: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:21.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:20 smithi017 ceph-mon[107556]: pgmap v1453: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:21.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:21.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:21.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:21.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:20 smithi189 ceph-mon[90992]: pgmap v1453: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:21.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:21.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:21.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:21.473 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:21.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:21.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:21.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:21.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:21.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:21.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:21.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:21.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:21.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:22.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:22.102 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:22.381 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:22.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:22.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:22.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:22.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:22.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:22.731 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.757 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:22.860 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:22.888 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:22.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:23.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:23.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:22 smithi017 ceph-mon[103001]: pgmap v1454: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:22 smithi017 ceph-mon[107556]: pgmap v1454: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:23.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:22 smithi189 ceph-mon[90992]: pgmap v1454: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:23.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:23.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.492 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:23.518 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:23.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:23.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:23.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:23.872 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:23.898 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:23.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:24.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:24.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:24.373 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:24.399 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:24.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:24.526 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:24.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:24.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:24.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:24.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:24.875 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:54:24.875 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:54:24.875 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:54:24.875 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:54:24.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:24 smithi017 ceph-mon[103001]: pgmap v1455: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:24.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:24 smithi017 ceph-mon[107556]: pgmap v1455: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:25.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:25.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:24 smithi189 ceph-mon[90992]: pgmap v1455: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:25.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:25.378 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:25.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:25.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:54:25.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:25.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:25.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.849 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:54:25.849 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:25.867 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:54:25.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:25.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:25.885 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:54:25.885 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:25.901 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:54:25.902 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:25.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:25.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.918 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:54:25.918 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:25.935 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:54:25.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:25.952 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:54:25.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:25.968 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:54:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:26.009 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.036 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:26.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:26.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:26.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:26.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:26.310 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:26.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:26.516 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:26.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:26.768 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:26.794 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:26.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:26.867 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:26 smithi017 ceph-mon[103001]: pgmap v1456: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:26.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3803645392' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:26.868 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:26 smithi017 ceph-mon[107556]: pgmap v1456: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:26.868 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3803645392' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:26.895 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:26.921 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:27.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:27.048 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:27.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.176 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:26 smithi189 ceph-mon[90992]: pgmap v1456: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3803645392' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:27.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:27.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:27.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:27.402 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.428 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:27] "GET /metrics HTTP/1.1" 200 34312 "" "Prometheus/2.51.0" 2024-09-13T14:54:27.530 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:27.555 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:27.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:27.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:27.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:27.804 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' created 2024-09-13T14:54:27.812 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:27.901 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4276309735' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4276309735' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:27.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:27.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.064 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:27 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4276309735' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:27 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:54:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:28.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:28.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:28.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:28.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:28.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:28.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.819 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:28.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:28.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:29.045 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:29.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:29.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:29.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[107556]: pgmap v1457: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:54:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[107556]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:54:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[103001]: pgmap v1457: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:54:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:28 smithi017 ceph-mon[103001]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:54:29.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:28 smithi189 ceph-mon[90992]: pgmap v1457: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:28 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:54:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:28 smithi189 ceph-mon[90992]: osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:54:29.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:29.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:29.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:29.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:29.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:29.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:29.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:29.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:29.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.975 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:30.072 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:29 smithi017 ceph-mon[103001]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:54:30.073 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:29 smithi017 ceph-mon[107556]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:54:30.076 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:30.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:30.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:29 smithi189 ceph-mon[90992]: osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:54:30.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:30.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:30.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:30.351 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:30.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:30.420 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:30.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:30.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:30.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:30.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:30.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-13T14:54:30.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:30.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:30.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[103001]: pgmap v1460: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:54:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[103001]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:54:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[107556]: pgmap v1460: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:54:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:30 smithi017 ceph-mon[107556]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:54:30.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:30.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:31.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.098 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:31.121 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:31.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:31.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:30 smithi189 ceph-mon[90992]: pgmap v1460: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:31.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:30 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:54:31.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:30 smithi189 ceph-mon[90992]: osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:54:31.323 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.345 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:31.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:31.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:31.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:31.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:31.818 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:54:31.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.821 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/847071543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/847071543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:31.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:31 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:31.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.948 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:31.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:31.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.072 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.195 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:32.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/847071543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:32.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:31 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:32.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:32.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:32.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:32.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:32.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:32.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:32.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:32.720 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:32.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:32.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:32.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:32.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.176 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[107556]: pgmap v1462: 99 pgs: 6 creating+peering, 9 unknown, 84 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:33.176 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:33.176 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[107556]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[103001]: pgmap v1462: 99 pgs: 6 creating+peering, 9 unknown, 84 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:33.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:32 smithi017 ceph-mon[103001]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:54:33.197 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:33.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:54:33.212 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:33.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:33.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:32 smithi189 ceph-mon[90992]: pgmap v1462: 99 pgs: 6 creating+peering, 9 unknown, 84 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:33.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:32 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:33.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:32 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/883542230' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:33.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:32 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:33.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:32 smithi189 ceph-mon[90992]: osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:54:33.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:33.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:33.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:33.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:33.696 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:33.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:33.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:33.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:33.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:33 smithi017 ceph-mon[103001]: pgmap v1464: 99 pgs: 99 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:34.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:33 smithi017 ceph-mon[107556]: pgmap v1464: 99 pgs: 99 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:34.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:33 smithi189 ceph-mon[90992]: pgmap v1464: 99 pgs: 99 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:34.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:34.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:34.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:34.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:54:34.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:34.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:34.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:34.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:34.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:35.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:35.096 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:35.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:35.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:35.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:35.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:35.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.726 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:35.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:35.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:35.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:35.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:36.076 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:36.102 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:36.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:36.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:36.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:36.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:36 smithi017 ceph-mon[103001]: pgmap v1465: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:36 smithi017 ceph-mon[107556]: pgmap v1465: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:36.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:36.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:36 smithi189 ceph-mon[90992]: pgmap v1465: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:36.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:36.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:36.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:36.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:37.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:37.122 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:37.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:37.276 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:37.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:37.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:37.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:37.635 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:37.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:37] "GET /metrics HTTP/1.1" 200 34312 "" "Prometheus/2.51.0" 2024-09-13T14:54:37.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:37.763 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:37.788 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:37.847 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:37.847 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:37.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:37.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:38.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.038 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.140 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.163 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.266 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:54:38.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:38.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:38.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:38.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:38.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:38.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:38.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:38.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:38.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:38 smithi017 ceph-mon[103001]: pgmap v1466: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:38.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2387640915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:38 smithi017 ceph-mon[107556]: pgmap v1466: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:38.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2387640915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.642 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.668 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:38 smithi189 ceph-mon[90992]: pgmap v1466: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:38.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2387640915' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:38.771 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.796 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:38.923 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:39.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:39.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:39.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:39.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.403 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:39.428 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:39.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:54:39.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:39.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:39.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:39.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:39.906 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:39.932 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:40.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:40.158 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.184 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:40.286 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.296 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:40.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:40.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:40.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:40.465 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:40.473 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-13T14:54:40.538 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.564 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:40.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[103001]: pgmap v1467: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2094829123' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[107556]: pgmap v1467: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2094829123' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:40 smithi189 ceph-mon[90992]: pgmap v1467: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:40 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2094829123' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:54:40.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.820 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.946 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:40.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:41.047 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:41.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:41.110 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:41.171 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:41.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:41.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:41.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:41.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:41.473 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:54:41.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:41.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[103001]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[107556]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:41 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:41.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:41 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:54:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:41 smithi189 ceph-mon[90992]: osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:54:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:41 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:41 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:41.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:41.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:41.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.942 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:41.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:41.975 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:42.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:42.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:42.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:42.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:42.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:42.539 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:42.564 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:42.666 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.690 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:42.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:42 smithi189 ceph-mon[90992]: pgmap v1469: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:42 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:54:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:42.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:42.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[103001]: pgmap v1469: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[107556]: pgmap v1469: 99 pgs: 99 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:42.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:42.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:42.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:42.942 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-13T14:54:42.944 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:54:42.944 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:54:42.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:42.990 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-13T14:54:42.990 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:54:42.990 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_372078 2024-09-13T14:54:42.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:42.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:54:43.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:43.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:43.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:43.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:43.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:43.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:43.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.735 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:43 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:43.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:43.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:43.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3715 3699 231.174 231.188 0.00460148 0.0043102 2024-09-13T14:54:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:43 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:54:44.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:44.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:44.191 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.227 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.329 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:44.366 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:44.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:44.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:44.602 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:54:44.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:44.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:44.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:44.738 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:44.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:44.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:44.912 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:44 smithi017 ceph-mon[103001]: pgmap v1470: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s 2024-09-13T14:54:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:44.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:44 smithi017 ceph-mon[107556]: pgmap v1470: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s 2024-09-13T14:54:44.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:44.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 15 7397 7382 230.67 230.188 0.00441814 0.00432681 2024-09-13T14:54:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:44 smithi189 ceph-mon[90992]: pgmap v1470: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s 2024-09-13T14:54:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:45.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:45.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:45.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:45.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:45.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:45.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:45.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:45.756 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:45.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:45.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:45.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:45 smithi017 ceph-mon[107556]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:54:45.857 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:45.857 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:45 smithi017 ceph-mon[103001]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:54:45.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:45.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:45.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 10825 10809 225.169 214.188 0.00453808 0.00443251 2024-09-13T14:54:45.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2274944749' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:45 smithi189 ceph-mon[90992]: osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:54:46.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:46.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:46.169 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:46.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:54:46.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:46.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:46.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:46.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:46 smithi017 ceph-mon[103001]: pgmap v1472: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s wr, 34 op/s 2024-09-13T14:54:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:46 smithi017 ceph-mon[107556]: pgmap v1472: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s wr, 34 op/s 2024-09-13T14:54:46.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:46.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:46.986 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:46.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 13846 13830 216.075 188.812 0.00465441 0.00462117 2024-09-13T14:54:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:46 smithi189 ceph-mon[90992]: pgmap v1472: 99 pgs: 99 active+clean; 22 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s wr, 34 op/s 2024-09-13T14:54:47.020 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:47.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:47.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:47.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:47.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:47.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:47.622 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:47] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-09-13T14:54:47.682 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:47.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:47.816 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:47.851 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:47.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:47.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 16813 16797 209.944 185.438 0.00695545 0.00475661 2024-09-13T14:54:47.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:48.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:48.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:48.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:48.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:48.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:48.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:48 smithi017 ceph-mon[103001]: pgmap v1473: 99 pgs: 99 active+clean; 230 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s 2024-09-13T14:54:48.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:48 smithi017 ceph-mon[107556]: pgmap v1473: 99 pgs: 99 active+clean; 230 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s 2024-09-13T14:54:48.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:48.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:48.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:48.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 19672 19656 204.732 178.688 0.00450159 0.00487805 2024-09-13T14:54:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:48 smithi189 ceph-mon[90992]: pgmap v1473: 99 pgs: 99 active+clean; 230 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s 2024-09-13T14:54:49.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:49.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:49.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:49.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:49.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:49.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:49.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:49.495 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:49.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:49.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88010 9.c 9:36a45f6d:::benchmark_data_smithi017_339564_object88009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:54:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.21577614, 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23807841599999999, 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88010 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377063+0000", 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8300000000000001e-06 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377118+0000", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4843999999999999e-05 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380264+0000", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003145994 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380275+0000", 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1649e-05 2024-09-13T14:54:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054572799999999999 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381007+0000", 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018597899999999999 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:54:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381155+0000", 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014763800000000001 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387265+0000", 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061102819999999999 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614900+0000", 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227635122 2024-09-13T14:54:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615054+0000", 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015373700000000001 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615094+0000", 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0738999999999997e-05 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615139+0000", 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4873999999999998e-05 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88013 9.c 9:3b76033e:::benchmark_data_smithi017_339564_object88012:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.21201613400001, 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23455462899999999, 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88013 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380822+0000", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3289999999999999e-06 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380883+0000", 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0396000000000002e-05 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380895+0000", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2823000000000001e-05 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380903+0000", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8860000000000004e-06 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384158+0000", 2024-09-13T14:54:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003254346 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384318+0000", 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016008400000000001 2024-09-13T14:54:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384451+0000", 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013359999999999999 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614962+0000", 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.230510786 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615189+0000", 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022717899999999999 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615283+0000", 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3936000000000003e-05 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615340+0000", 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6261999999999998e-05 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615376+0000", 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6001999999999999e-05 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88014 9.c 9:30066bc1:::benchmark_data_smithi017_339564_object88013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.211920255, 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23458493899999999, 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88014 2024-09-13T14:54:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380969+0000", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1617999999999999e-05 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380978+0000", 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4979999999999995e-06 2024-09-13T14:54:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380985+0000", 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387301+0000", 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063155529999999998 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387451+0000", 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150414 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387614+0000", 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016278399999999999 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615015+0000", 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227400877 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615195+0000", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017990500000000001 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615419+0000", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224207 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615466+0000", 2024-09-13T14:54:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7579e-05 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615502+0000", 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5425e-05 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:88018 9.4 e381/376 9:2920a091:::benchmark_data_smithi017_339564_object88017:head v 381'5555, mlcod=381'5553)", 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.21011033100001, 2024-09-13T14:54:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23245458799999999, 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:54:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382728+0000", 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6099999999999999e-07 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382749+0000", 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1486000000000001e-05 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382760+0000", 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1107999999999999e-05 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382767+0000", 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8820000000000003e-06 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384135+0000", 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013674410000000001 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384156+0000", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1027999999999999e-05 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615151+0000", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23099550899999999 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615181+0000", 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0173000000000001e-05 2024-09-13T14:54:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88020 9.c 9:32f382c7:::benchmark_data_smithi017_339564_object88019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.20988465799999, 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23295475700000001, 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88020 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:54:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382954+0000", 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1620000000000001e-06 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383001+0000", 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7929000000000001e-05 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383013+0000", 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1324e-05 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383020+0000", 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5340000000000002e-06 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390514+0000", 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074939789999999996 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390651+0000", 2024-09-13T14:54:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013669299999999999 2024-09-13T14:54:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390787+0000", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135773 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615200+0000", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22441285599999999 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615234+0000", 2024-09-13T14:54:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4634000000000001e-05 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615794+0000", 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055964299999999997 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615859+0000", 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5318e-05 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615907+0000", 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7911999999999998e-05 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88023 9.2 9:452883ed:::benchmark_data_smithi017_339564_object88022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.20429770300001, 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.226308658, 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88023 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388539+0000", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388541+0000", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4589999999999999e-06 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388592+0000", 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1138999999999999e-05 2024-09-13T14:54:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388604+0000", 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2330999999999999e-05 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388612+0000", 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2749999999999998e-06 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388666+0000", 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4552000000000002e-05 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388799+0000", 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013261499999999999 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.388937+0000", 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137942 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.392307+0000", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033704109999999998 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393094+0000", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078649099999999995 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393918+0000", 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082484500000000005 2024-09-13T14:54:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393974+0000", 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5798999999999999e-05 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614848+0000", 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22087379900000001 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88025 9.8 9:1a0cff2e:::benchmark_data_smithi017_339564_object88024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.20213241600001, 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23531686499999999, 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88025 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390706+0000", 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3009999999999999e-06 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390728+0000", 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1959000000000001e-05 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390741+0000", 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2892e-05 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390748+0000", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390808+0000", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0133000000000002e-05 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390954+0000", 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014599599999999999 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391077+0000", 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012244 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621755+0000", 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23067821399999999 2024-09-13T14:54:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621803+0000", 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8112999999999998e-05 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625781+0000", 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039782510000000004 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625827+0000", 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5982000000000001e-05 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626021+0000", 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019408899999999999 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:88013 9.c e381/376 ondisk, result = 0)", 2024-09-13T14:54:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.20192824, 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22407027400000001, 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390909+0000", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390910+0000", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0500000000000002e-07 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390913+0000", 2024-09-13T14:54:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.76e-06 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390921+0000", 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0080000000000002e-06 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390928+0000", 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1419999999999998e-06 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614947+0000", 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.224019412 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614955+0000", 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0760000000000003e-06 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614979+0000", 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3971000000000001e-05 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46712.0:88014 9.c e381/376 ondisk, result = 0)", 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.20168220799999, 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22386584100000001, 2024-09-13T14:54:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391155+0000", 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391156+0000", 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5600000000000005e-07 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391159+0000", 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.49e-06 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391167+0000", 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3850000000000002e-06 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391173+0000", 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0000000000000002e-06 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615005+0000", 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.223831266 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615011+0000", 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0759999999999999e-06 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615021+0000", 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8679999999999994e-06 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88029 9.8 9:1879118d:::benchmark_data_smithi017_339564_object88028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.19982148899999, 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.233198557, 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88029 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393017+0000", 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.37e-06 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393040+0000", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2705e-05 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393052+0000", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1895e-05 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393061+0000", 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:54:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614926+0000", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.221865756 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615166+0000", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000239127 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615958+0000", 2024-09-13T14:54:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079216400000000004 2024-09-13T14:54:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.657 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621846+0000", 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058882079999999998 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625766+0000", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039203490000000001 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626034+0000", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026819300000000001 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626171+0000", 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136966 2024-09-13T14:54:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626214+0000", 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2703000000000001e-05 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:88027 9.f e381/376 9:ff7c9408:::benchmark_data_smithi017_339564_object88026:head v 381'5566, mlcod=381'5564)", 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.199583397, 2024-09-13T14:54:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22187700399999999, 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393254+0000", 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2099999999999996e-07 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393408+0000", 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153683 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393418+0000", 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0006999999999999e-05 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393424+0000", 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4989999999999999e-06 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393492+0000", 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7174000000000004e-05 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393511+0000", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9831999999999999e-05 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615091+0000", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.221579844 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615131+0000", 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9344000000000001e-05 2024-09-13T14:54:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88034 9.8 9:16b0f031:::benchmark_data_smithi017_339564_object88033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 226.19496332200001, 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.228835232, 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88034 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397874+0000", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397876+0000", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.922e-06 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.397940+0000", 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4468999999999995e-05 2024-09-13T14:54:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614828+0000", 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21688771200000001 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614841+0000", 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3155e-05 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622019+0000", 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071777079999999997 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622179+0000", 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016065799999999999 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.622695+0000", 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051556699999999998 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626221+0000", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035260869999999998 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626332+0000", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110741 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626603+0000", 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000271363 2024-09-13T14:54:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626670+0000", 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6789e-05 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626709+0000", 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9060999999999998e-05 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.81197416500001, 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30486941899999997, 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:54:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.673 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:54:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780865+0000", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.629e-06 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780911+0000", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6511999999999998e-05 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780922+0000", 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1454000000000001e-05 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780930+0000", 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9019999999999999e-06 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781098+0000", 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016806699999999999 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781253+0000", 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015428100000000001 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781490+0000", 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023734 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789140+0000", 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076497320000000002 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800223+0000", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011082834999999999 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085715+0000", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28549267499999997 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085732+0000", 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6991999999999999e-05 2024-09-13T14:54:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.2 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.80743594200001, 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30060562000000002, 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785402+0000", 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8100000000000003e-07 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785421+0000", 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9623000000000002e-05 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785429+0000", 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785435+0000", 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789170+0000", 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037350439999999999 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789333+0000", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162893 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789491+0000", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015807199999999999 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797726+0000", 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082348890000000004 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805320+0000", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075942850000000001 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085994+0000", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28067408599999999 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.086007+0000", 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2286000000000001e-05 2024-09-13T14:54:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.2 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.80574849300001, 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30002994399999999, 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787090+0000", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.561e-06 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788801+0000", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001711113 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788814+0000", 2024-09-13T14:54:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3242e-05 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788822+0000", 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7109999999999995e-06 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794251+0000", 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054290520000000002 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794407+0000", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015591100000000001 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794559+0000", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015160600000000001 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801869+0000", 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007309973 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087105+0000", 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28523667899999999 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087118+0000", 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3096e-05 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.67484698300001, 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90794274900000005, 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917993+0000", 2024-09-13T14:54:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5560000000000002e-06 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918027+0000", 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4338000000000002e-05 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918056+0000", 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9411e-05 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918071+0000", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4154999999999999e-05 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087146+0000", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16907546600000001 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087156+0000", 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0069e-05 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825063+0000", 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.737906583 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825933+0000", 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087017099999999999 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.12 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.67474015299999, 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92020213900000003, 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3300000000000003e-07 2024-09-13T14:54:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918118+0000", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0727e-05 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0434999999999999e-05 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918134+0000", 2024-09-13T14:54:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5589999999999996e-06 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087181+0000", 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16904727899999999 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087188+0000", 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7519999999999999e-06 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838221+0000", 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.751033124 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838299+0000", 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7830000000000005e-05 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.12 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:54:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.674695883, 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92069096500000003, 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918142+0000", 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6800000000000001e-07 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918175+0000", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3642000000000001e-05 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918182+0000", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3920000000000002e-06 2024-09-13T14:54:49.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087211+0000", 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16902355999999999 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087217+0000", 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.835e-06 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838755+0000", 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.75153752500000004 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838832+0000", 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7414000000000001e-05 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95200 9.18 9:1838bbdd:::benchmark_data_smithi017_339564_object95199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:54:49.705 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 219.40474133699999, 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65754950099999998, 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95200 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188098+0000", 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4040000000000002e-06 2024-09-13T14:54:49.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188129+0000", 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1032999999999999e-05 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188155+0000", 2024-09-13T14:54:49.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6322999999999999e-05 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188169+0000", 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3512999999999999e-05 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188244+0000", 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5078000000000002e-05 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188258+0000", 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.347e-05 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839583+0000", 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65132497099999997 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839664+0000", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1942999999999998e-05 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839771+0000", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010627 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842419+0000", 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026485020000000001 2024-09-13T14:54:49.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842538+0000", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119218 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845548+0000", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030097520000000001 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845600+0000", 2024-09-13T14:54:49.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2218e-05 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845645+0000", 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4805999999999998e-05 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:159241 9.2 9:447bd675:::benchmark_data_smithi017_339564_object39768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:52:23.942528+0000", 2024-09-13T14:54:49.713 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 145.65030874600001, 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23091978899999999, 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 159241 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942528+0000", 2024-09-13T14:54:49.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942528+0000", 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942529+0000", 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1600000000000002e-07 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:54:49.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942531+0000", 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2010000000000002e-06 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942537+0000", 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0630000000000002e-06 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:54:49.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942542+0000", 2024-09-13T14:54:49.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9810000000000003e-06 2024-09-13T14:54:49.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.942948+0000", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040567000000000001 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.943093+0000", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014567099999999999 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.943571+0000", 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047735900000000001 2024-09-13T14:54:49.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.947091+0000", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035206510000000001 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:23.947174+0000", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3112e-05 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.173316+0000", 2024-09-13T14:54:49.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22614171199999999 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.173387+0000", 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0438999999999994e-05 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:52:24.173448+0000", 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1513999999999994e-05 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.720 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.721 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:54:49.721 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:54:49.721 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:54:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.722 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:49.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:49.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.862 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:49.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/168688376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:49 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/168688376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:49 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:49.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:49.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 22273 22257 198.706 162.562 0.00418574 0.00502632 2024-09-13T14:54:49.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/168688376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:49 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:50.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:50.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:50.373 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.408 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:54:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:54:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:50.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:50.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:50.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[103001]: pgmap v1474: 99 pgs: 99 active+clean; 992 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:54:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[103001]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[107556]: pgmap v1474: 99 pgs: 99 active+clean; 992 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:50 smithi017 ceph-mon[107556]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:54:50.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:50.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:50.927 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.962 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 24903 24887 194.413 164.375 0.00542696 0.00513877 2024-09-13T14:54:51.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:50 smithi189 ceph-mon[90992]: pgmap v1474: 99 pgs: 99 active+clean; 992 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2024-09-13T14:54:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:50 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:50 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:50 smithi189 ceph-mon[90992]: osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:54:51.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:51.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:51.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:51.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:51.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:51.509 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:51.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:51.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:51.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:51.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:51.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 27316 27300 189.567 150.812 0.0063051 0.00526941 2024-09-13T14:54:52.026 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:52.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:52.164 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:52.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:52.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:52.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:52.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:52.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:52.615 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:52.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:52.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:52.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:52 smithi017 ceph-mon[107556]: pgmap v1476: 99 pgs: 99 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:54:52.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:52 smithi017 ceph-mon[103001]: pgmap v1476: 99 pgs: 99 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:54:52.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:54:52.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 29807 29791 186.178 155.688 0.00451301 0.00536597 2024-09-13T14:54:52.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:52 smithi189 ceph-mon[90992]: pgmap v1476: 99 pgs: 99 active+clean; 1.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:54:53.053 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:53.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:53.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:53.220 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:53.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.353 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.420 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:53.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:53.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:53.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:53.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:53.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:53.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:53.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:53.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:53.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:53.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 31918 31902 181.246 131.938 0.00414035 0.00551189 2024-09-13T14:54:54.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:54.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:54.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:54.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:54.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:54.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:54.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.494 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:54.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:54.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:54 smithi189 ceph-mon[90992]: pgmap v1477: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-09-13T14:54:54.772 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.874 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:54.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:54 smithi017 ceph-mon[103001]: pgmap v1477: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-09-13T14:54:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:54 smithi017 ceph-mon[107556]: pgmap v1477: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 203 MiB/s wr, 3.25k op/s 2024-09-13T14:54:54.991 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 15 33943 33928 176.693 126.625 0.00713113 0.00565347 2024-09-13T14:54:55.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:55.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:54:55.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:55.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:55.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:55.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.440 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.542 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:55.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.708 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.810 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:55.843 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:55.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 35607 35591 171.096 103.938 0.0148459 0.00583934 2024-09-13T14:54:56.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:56.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:56.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:56.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:54:56.327 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:56.355 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:54:56.355 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:56.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.397 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:54:56.397 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:56.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:54:56.451 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:56.490 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:54:56.490 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:56.533 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:54:56.534 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:56.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:56.571 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:54:56.572 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:56.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:56.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:54:56.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:56.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:56.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:56.819 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.853 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:56.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:56.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:56 smithi017 ceph-mon[103001]: pgmap v1478: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s 2024-09-13T14:54:56.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4071565216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:56.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:56 smithi017 ceph-mon[107556]: pgmap v1478: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s 2024-09-13T14:54:56.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4071565216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:56.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:56.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:56.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 37205 37189 166.008 99.875 0.00888002 0.00601754 2024-09-13T14:54:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:56 smithi189 ceph-mon[90992]: pgmap v1478: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s 2024-09-13T14:54:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4071565216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:57.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:57.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:57.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:57.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:57.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:57.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:57.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:57.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:57.670 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:54:57] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-09-13T14:54:57.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3500985450' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:57.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:57.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:57.675 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:54:57.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:57.879 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:57.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3500985450' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:57.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 38550 38534 160.545 84.0625 0.00820546 0.00622346 2024-09-13T14:54:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3500985450' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:58.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:58.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:58.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:58.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:58.441 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:58.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:58.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:58.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:58.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.754 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:58.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:58.892 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[103001]: pgmap v1479: 99 pgs: 99 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-09-13T14:54:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[103001]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:54:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[107556]: pgmap v1479: 99 pgs: 99 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-09-13T14:54:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:54:58 smithi017 ceph-mon[107556]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:54:58.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 40284 40268 157.283 108.375 0.00384284 0.00635217 2024-09-13T14:54:58.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:58 smithi189 ceph-mon[90992]: pgmap v1479: 99 pgs: 99 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2024-09-13T14:54:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:54:58 smithi189 ceph-mon[90992]: osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:54:59.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:59.131 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:59.157 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:59.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:59.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:59.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:59.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:59.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:59.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:59.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:59.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:54:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:59.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:59.781 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:59.882 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:59.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:54:59.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 41184 41168 151.34 56.25 0.00936681 0.00660152 2024-09-13T14:55:00.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.052 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:00.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:00.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.290 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:00.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:00.326 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:00.427 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.463 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:00.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:00.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.591 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:00.591 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.604 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:00.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:00.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:00.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:00.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:00.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:00.876 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:00 smithi189 ceph-mon[90992]: pgmap v1481: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-13T14:55:00.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:00.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 43010 42994 149.272 114.125 0.00764295 0.00669363 2024-09-13T14:55:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.014 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:01.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:01.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:01.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:00 smithi017 ceph-mon[103001]: pgmap v1481: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-13T14:55:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:00 smithi017 ceph-mon[107556]: pgmap v1481: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-13T14:55:01.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.294 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.306 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:55:01.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.438 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:01.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:01.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:01.810 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:01.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:01.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 44679 44663 146.905 104.312 0.0124511 0.00680231 2024-09-13T14:55:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.008 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:02.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.109 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.145 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:02.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.423 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:02.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.705 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:02.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:02.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:55:02.992354+0000 min lat: 0.00143583 max lat: 0.28921 avg lat: 0.00690057 2024-09-13T14:55:02.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:55:02.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 46358 46342 144.806 104.938 0.00799748 0.00690057 2024-09-13T14:55:03.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:03.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:03.153 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:02 smithi189 ceph-mon[90992]: pgmap v1482: 99 pgs: 99 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:55:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:02 smithi017 ceph-mon[103001]: pgmap v1482: 99 pgs: 99 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:55:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:02 smithi017 ceph-mon[107556]: pgmap v1482: 99 pgs: 99 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:55:03.205 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:03.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:03.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:03.500 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:03.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:03.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:03.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:03.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:55:03.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:03.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:03.993 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 48088 48072 143.052 108.125 0.00752497 0.0069858 2024-09-13T14:55:04.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:04.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:04.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:04.251 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:04.251 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:04.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:04.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:04.394 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.496 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:04.533 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:04.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:04.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:04.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:04.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:04.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: pgmap v1483: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: pgmap v1483: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:04.993 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 49775 49759 141.342 105.438 0.0121986 0.00706976 2024-09-13T14:55:05.053 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.090 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:05.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: pgmap v1483: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:05.239 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:05.284 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:55:05.331 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:05.334 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.367 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:05.606 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:05.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:05.718 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:05.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:05.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:05.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:05.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:05.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:05.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:05.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 51522 51506 139.944 109.188 0.0104435 0.00714011 2024-09-13T14:55:06.019 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:06.060 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:06.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:06.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:06.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:06.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:06.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:06.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:06.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:06.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:06.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:06.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.775 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:06.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:06.876 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:06.908 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:06.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 53219 53203 138.532 106.062 0.0110252 0.00721367 2024-09-13T14:55:07.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:07.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:07.074 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:06 smithi189 ceph-mon[90992]: pgmap v1484: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:07.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:06 smithi017 ceph-mon[103001]: pgmap v1484: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:06 smithi017 ceph-mon[107556]: pgmap v1484: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:55:07.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:07.329 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:07.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:07.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:07.518 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:07] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-09-13T14:55:07.565 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:07.585 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:07.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:07.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:07.824 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:07.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:07.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:07.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 54772 54756 136.873 97.0625 0.00385899 0.0073002 2024-09-13T14:55:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.995 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:07.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:08.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:08.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:08.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:08.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.379 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:08.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:08.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:08.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:08.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:08.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:08.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:08.813 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:55:08.814 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:55:08.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:08.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:08.916 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:08.916 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:08.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 56117 56101 134.841 84.0625 0.00583456 0.00741054 2024-09-13T14:55:08.996 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:08 smithi189 ceph-mon[90992]: pgmap v1485: 99 pgs: 99 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:55:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4263155689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:08 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/674950868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:08 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:09.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[103001]: pgmap v1485: 99 pgs: 99 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4263155689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/674950868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[107556]: pgmap v1485: 99 pgs: 99 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4263155689' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/674950868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:08 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:09.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.515 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.549 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:09.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:09.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:09.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:55:09.785 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:09.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:09.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:09.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:09.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 57777 57761 133.689 103.75 0.0110911 0.00747486 2024-09-13T14:55:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:09 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:09 smithi189 ceph-mon[90992]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:55:10.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:10.135 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[103001]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:09 smithi017 ceph-mon[107556]: osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:55:10.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:10.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:10.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:55:10.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:10.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:10.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:10.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:10.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:10.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:10.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:10.742 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:10.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.778 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:10.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:10.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:10.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 59006 58990 131.658 76.8125 0.00627802 0.00759013 2024-09-13T14:55:11.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:11.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:11.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:11.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:10 smithi017 ceph-mon[103001]: pgmap v1487: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:10 smithi017 ceph-mon[107556]: pgmap v1487: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:11.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:10 smithi189 ceph-mon[90992]: pgmap v1487: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:11.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:11.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:11.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:11.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:11.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:11.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:11.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:11.740 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:11.842 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:11.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:11.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:11 smithi017 ceph-mon[107556]: pgmap v1488: 99 pgs: 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:11.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:11 smithi017 ceph-mon[103001]: pgmap v1488: 99 pgs: 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:11.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:11.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 60333 60317 129.978 82.9375 0.00948451 0.00768852 2024-09-13T14:55:12.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:11 smithi189 ceph-mon[90992]: pgmap v1488: 99 pgs: 99 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:55:12.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:12.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:12.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:12.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:12.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:12.395 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:12.429 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:12.531 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.556 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:12.566 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.625 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:55:12.625 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:12.664 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:55:12.664 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:12.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:12.701 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:55:12.702 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:12.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:12.737 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:55:12.738 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:12.775 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:55:12.775 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:12.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:55:12.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:12.817 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.838 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:55:12.838 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:12.845 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.869 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:55:12.870 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:12.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2496296491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:12.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2496296491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:12.989 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 61747 61731 128.591 88.375 0.00441092 0.00777063 2024-09-13T14:55:13.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2496296491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:13.091 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.126 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.266 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.402 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.505 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:13.790 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:13.831 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:13.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:13 smithi017 ceph-mon[107556]: pgmap v1489: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:13 smithi017 ceph-mon[103001]: pgmap v1489: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:13.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:55:13.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.994 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:13 smithi189 ceph-mon[90992]: pgmap v1489: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:13.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 63149 63133 127.269 87.625 0.00756079 0.00785174 2024-09-13T14:55:14.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:14.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:14.215 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:14.400 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:14.501 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:14.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:14.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.667 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:64251 10.b 10:d24e6fe1:::benchmark_data_smithi017_372078_object64250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.746068+0000", 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0059014949999999997, 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059338180000000004, 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:14.756 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 64251 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746068+0000", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746068+0000", 2024-09-13T14:55:14.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746066+0000", 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746133+0000", 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7162000000000005e-05 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746153+0000", 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9542999999999999e-05 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746160+0000", 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7340000000000002e-06 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746206+0000", 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6465999999999999e-05 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.759 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746311+0000", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000104893 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746468+0000", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015705 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749348+0000", 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028798579999999999 2024-09-13T14:55:14.760 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:64254 10.0 10:09bfe663:::benchmark_data_smithi017_372078_object64253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.746703+0000", 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0052667900000000004, 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0053812210000000003, 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:55:14.761 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 64254 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746703+0000", 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746703+0000", 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746702+0000", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746800+0000", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7607000000000002e-05 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746810+0000", 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0657e-05 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746815+0000", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2850000000000002e-06 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746861+0000", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5989999999999998e-05 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746985+0000", 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123065 2024-09-13T14:55:14.764 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.747117+0000", 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013231899999999999 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749323+0000", 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002206511 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.765 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.37095.0:64248 10.e e457/449 10:7baca22c:::benchmark_data_smithi017_372078_object64247:head v 457'4043, mlcod=457'4040)", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.748796+0000", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0031734649999999999, 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0033672699999999999, 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748796+0000", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748796+0000", 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748796+0000", 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748825+0000", 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9527999999999999e-05 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748833+0000", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.401e-06 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.748837+0000", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9829999999999998e-06 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.750004+0000", 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0011676429999999999 2024-09-13T14:55:14.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.750022+0000", 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7455000000000001e-05 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:64257 10.f 10:fff9c5f7:::benchmark_data_smithi017_372078_object64256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.749402+0000", 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.002567167, 2024-09-13T14:55:14.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028179210000000001, 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 64257 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749402+0000", 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749402+0000", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749401+0000", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749468+0000", 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6820000000000001e-05 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749481+0000", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2612000000000001e-05 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749486+0000", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3310000000000003e-06 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.750608+0000", 2024-09-13T14:55:14.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001122411 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.750732+0000", 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123385 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.750875+0000", 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014351199999999999 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop_reply(client.37095.0:64247 10.f e457/449)", 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.749790+0000", 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.002179238, 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0024899900000000001, 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "queued for pg", 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749790+0000", 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749790+0000", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749790+0000", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749796+0000", 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3069999999999999e-06 2024-09-13T14:55:14.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749808+0000", 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.15e-05 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.749812+0000", 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5700000000000003e-06 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:64245 10.a 10:5eb86ada:::benchmark_data_smithi017_372078_object64244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.743235+0000", 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0087342510000000002, 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0090924869999999998, 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 64245 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.777 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743235+0000", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743235+0000", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743234+0000", 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743397+0000", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00016298699999999999 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743411+0000", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3807e-05 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743416+0000", 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4219999999999999e-06 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743667+0000", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025097200000000002 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743771+0000", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010417 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743902+0000", 2024-09-13T14:55:14.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013055799999999999 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746568+0000", 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002666009 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:64247 10.f 10:f90a8ab8:::benchmark_data_smithi017_372078_object64246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:14.743431+0000", 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0085385619999999995, 2024-09-13T14:55:14.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0089682779999999997, 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 64247 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743431+0000", 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743431+0000", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743430+0000", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743481+0000", 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0809000000000001e-05 2024-09-13T14:55:14.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743490+0000", 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.1460000000000002e-06 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.743497+0000", 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9290000000000001e-06 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:14.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746000+0000", 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0025023490000000001 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746112+0000", 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000112438 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:14.746246+0000", 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013406 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.785 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:14.786 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.786 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:14.786 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:55:14.786 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 7 2024-09-13T14:55:14.786 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:55:14.786 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.832 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.934 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 64630 64614 126.184 92.5625 0.00965947 0.00791992 2024-09-13T14:55:15.071 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:15.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:15.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:15.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:15.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:15.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:15.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:15.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:15.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:15.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:15.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:15.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:15.886 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:15.907 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:15.994 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 65916 65900 124.796 80.375 0.00859685 0.00800816 2024-09-13T14:55:16.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:16.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:16.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:16.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:16.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:16.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:16.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:16.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:16.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:16 smithi189 ceph-mon[90992]: pgmap v1490: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:16.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:16.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:16.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:55:16.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:16 smithi017 ceph-mon[103001]: pgmap v1490: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:16 smithi017 ceph-mon[107556]: pgmap v1490: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:55:16.995 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 67352 67336 123.762 89.75 0.00791755 0.00807497 2024-09-13T14:55:17.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:17.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:17.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:17.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:17.340 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:17.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:17.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:17.482 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:17.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:17.606 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:17] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-09-13T14:55:17.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:17.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:17.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:17.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:17.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.959 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 68717 68701 122.663 85.3125 0.0060606 0.00814762 2024-09-13T14:55:18.061 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:18.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:18.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:18.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.239 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:18.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:18.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:18.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.483 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:55:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:18.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:18.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[103001]: pgmap v1491: 99 pgs: 99 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:55:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2427924284' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[107556]: pgmap v1491: 99 pgs: 99 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:55:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2427924284' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:18 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:18 smithi189 ceph-mon[90992]: pgmap v1491: 99 pgs: 99 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:55:18.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2427924284' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:18 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:55:18.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:18.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:18.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 69865 69849 121.248 71.75 0.00502529 0.00824216 2024-09-13T14:55:19.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:19.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:19.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.186 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:19.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:19.288 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:19.328 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:19.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.472 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:19.503 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:55:19.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:19.614 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:55:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[103001]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:55:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:55:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[107556]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:55:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:19 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:19.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.739 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:19 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:55:19.739 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:19 smithi189 ceph-mon[90992]: osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:55:19.739 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:19 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:19.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.786 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:55:19.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:19.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:19.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:19.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 71270 71254 120.345 87.8125 0.00940211 0.00830459 2024-09-13T14:55:20.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:20.051 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:55:20.051 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:55:20.051 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:55:20.051 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:55:20.051 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:55:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:20.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:20.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.257 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:20.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:20.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:20.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:20.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:20.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:20.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-mon[90992]: pgmap v1493: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-13T14:55:20.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:20.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:55:20.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:20.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:20 smithi189 ceph-mon[90992]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:55:20.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:20.748 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:55:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:20.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:20.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:20.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[107556]: pgmap v1493: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-13T14:55:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[107556]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[103001]: pgmap v1493: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4248520480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:20.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:20 smithi017 ceph-mon[103001]: osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:55:20.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 72628 72612 119.411 84.875 0.009521 0.00836943 2024-09-13T14:55:21.010 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:21.046 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:21.147 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:21.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:21.216 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:21.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:21.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:21.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:21.592 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:55:21.593 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:21.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:21.599 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:21.607 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:55:21.607 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:21.624 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:55:21.625 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:21.658 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:55:21.658 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:21.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:55:21.695 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:21.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.732 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:55:21.732 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:21.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.763 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:55:21.763 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:21.794 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:55:21.794 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:21.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:21.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.993 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:21.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 73714 73698 118.09 67.875 0.0108478 0.00846276 2024-09-13T14:55:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:22.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:22.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:22.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:22.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:22.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:22.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:22.443 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:22.545 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:22.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:22.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:22.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:22.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:22.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:22 smithi017 ceph-mon[103001]: pgmap v1495: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-09-13T14:55:22.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3822228845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:22.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:22 smithi017 ceph-mon[107556]: pgmap v1495: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-09-13T14:55:22.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3822228845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:22.816 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:22 smithi189 ceph-mon[90992]: pgmap v1495: 99 pgs: 99 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-09-13T14:55:22.816 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3822228845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:22.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:22.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:22.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:55:22.995976+0000 min lat: 0.00143583 max lat: 0.28921 avg lat: 0.00853132 2024-09-13T14:55:22.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:55:22.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 74998 74982 117.144 80.25 0.00908689 0.00853132 2024-09-13T14:55:22.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:22.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:23.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:55:23.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:23.135 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:23.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:23.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.272 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:23.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:23.375 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.412 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:23.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:23.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:23.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:23.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:23.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:23.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:23.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:23.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.984 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:23.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:23.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 76354 76338 116.353 84.75 0.0119405 0.00858975 2024-09-13T14:55:24.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.122 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.224 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:24.262 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:24.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:24.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:24.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:24.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:24.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:24.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:24.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:24.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:24 smithi017 ceph-mon[103001]: pgmap v1496: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:55:24.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:24 smithi017 ceph-mon[107556]: pgmap v1496: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:55:24.617 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:24.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:24.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:24.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:24.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:24.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:24.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:24.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 77271 77255 114.948 57.3125 0.00598748 0.00869497 2024-09-13T14:55:25.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:24 smithi189 ceph-mon[90992]: pgmap v1496: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:55:25.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:25.061 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:25.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:25.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:25.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:25.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:25.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:25.362 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:25.363 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:25.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:25.473 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:25.575 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.613 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:25.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.686 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:55:25.690 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:55:25.702 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:25.715 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:25.741 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:25.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:25.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:25.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 78378 78362 113.883 69.1875 0.0138653 0.00877487 2024-09-13T14:55:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:55:26.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.178 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:26.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:26.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:26.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:26.514 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:26.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:26.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:26.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:26.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:26 smithi189 ceph-mon[90992]: from='client.47012 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:26.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:26 smithi189 ceph-mon[90992]: pgmap v1497: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:55:26.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:26.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:26.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:26 smithi017 ceph-mon[103001]: from='client.47012 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:26.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:26 smithi017 ceph-mon[103001]: pgmap v1497: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:55:26.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:26 smithi017 ceph-mon[107556]: from='client.47012 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:26.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:26 smithi017 ceph-mon[107556]: pgmap v1497: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:55:26.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 79655 79639 113.109 79.8125 0.00858997 0.00883627 2024-09-13T14:55:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:27.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:27.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:27.278 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:27.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:27.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:27.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:27.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:27] "GET /metrics HTTP/1.1" 200 36719 "" "Prometheus/2.51.0" 2024-09-13T14:55:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.789 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:27.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:27.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:27.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 80868 80852 112.28 75.8125 0.0195205 0.00890051 2024-09-13T14:55:28.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:28.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:28.178 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1496,"stamp":"2024-09-13T14:55:23.419477+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4896266189,"num_objects":74666,"num_object_clones":0,"num_object_copies":223998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74666,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":75545,"num_write_kb":4789455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":8394258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76770,"ondisk_log_size":76770,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21095676,"kb_used_data":14201964,"kb_used_omap":44,"kb_used_meta":6893587,"kb_avail":728865540,"statfs":{"total":767960285184,"available":746358312960,"internally_reserved":0,"allocated":14542811136,"data_stored":14523367203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7059033335},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[11,1,6,2,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":874119168,"num_objects":13338,"num_object_clones":0,"num_object_copies":40014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13338,"num_write_kb":853632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002041"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":804,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.491754+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:55:18.491754+0000","last_peered":"2024-09-13T14:55:18.491754+0000","last_clean":"2024-09-13T14:55:18.491754+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:55:18.491754+0000","last_undegraded":"2024-09-13T14:55:18.491754+0000","last_fullsized":"2024-09-13T14:55:18.491754+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":457,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480988+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:55:18.480988+0000","last_peered":"2024-09-13T14:55:18.480988+0000","last_clean":"2024-09-13T14:55:18.480988+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:55:18.480988+0000","last_undegraded":"2024-09-13T14:55:18.480988+0000","last_fullsized":"2024-09-13T14:55:18.480988+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":386,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478359+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:55:18.478359+0000","last_peered":"2024-09-13T14:55:18.478359+0000","last_clean":"2024-09-13T14:55:18.478359+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:55:18.478359+0000","last_undegraded":"2024-09-13T14:55:18.478359+0000","last_fullsized":"2024-09-13T14:55:18.478359+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515611+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:55:19.515611+0000","last_peered":"2024-09-13T14:55:19.515611+0000","last_clean":"2024-09-13T14:55:19.515611+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:55:19.515611+0000","last_undegraded":"2024-09-13T14:55:19.515611+0000","last_fullsized":"2024-09-13T14:55:19.515611+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515734+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:55:19.515734+0000","last_peered":"2024-09-13T14:55:19.515734+0000","last_clean":"2024-09-13T14:55:19.515734+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:55:19.515734+0000","last_undegraded":"2024-09-13T14:55:19.515734+0000","last_fullsized":"2024-09-13T14:55:19.515734+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":426,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.484045+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:55:18.484045+0000","last_peered":"2024-09-13T14:55:18.484045+0000","last_clean":"2024-09-13T14:55:18.484045+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:55:18.484045+0000","last_undegraded":"2024-09-13T14:55:18.484045+0000","last_fullsized":"2024-09-13T14:55:18.484045+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":803,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.492028+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:55:18.492028+0000","last_peered":"2024-09-13T14:55:18.492028+0000","last_clean":"2024-09-13T14:55:18.492028+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:55:18.492028+0000","last_undegraded":"2024-09-13T14:55:18.492028+0000","last_fullsized":"2024-09-13T14:55:18.492028+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":488,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486399+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:55:18.486399+0000","last_peered":"2024-09-13T14:55:18.486399+0000","last_clean":"2024-09-13T14:55:18.486399+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:55:18.486399+0000","last_undegraded":"2024-09-13T14:55:18.486399+0000","last_fullsized":"2024-09-13T14:55:18.486399+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"459'4674","reported_seq":4692,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:23.336594+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:55:23.336594+0000","last_peered":"2024-09-13T14:55:23.336594+0000","last_clean":"2024-09-13T14:55:23.336594+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:55:23.336594+0000","last_undegraded":"2024-09-13T14:55:23.336594+0000","last_fullsized":"2024-09-13T14:55:23.336594+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4674,"log_dups_size":0,"ondisk_log_size":4674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4674,"num_write_kb":299136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":480,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481899+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:55:18.481899+0000","last_peered":"2024-09-13T14:55:18.481899+0000","last_clean":"2024-09-13T14:55:18.481899+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:55:18.481899+0000","last_undegraded":"2024-09-13T14:55:18.481899+0000","last_fullsized":"2024-09-13T14:55:18.481899+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":371,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477551+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:55:18.477551+0000","last_peered":"2024-09-13T14:55:18.477551+0000","last_clean":"2024-09-13T14:55:18.477551+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:55:18.477551+0000","last_undegraded":"2024-09-13T14:55:18.477551+0000","last_fullsized":"2024-09-13T14:55:18.477551+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"459'4770","reported_seq":4788,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.779090+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:55:22.779090+0000","last_peered":"2024-09-13T14:55:22.779090+0000","last_clean":"2024-09-13T14:55:22.779090+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:55:22.779090+0000","last_undegraded":"2024-09-13T14:55:22.779090+0000","last_fullsized":"2024-09-13T14:55:22.779090+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312606720,"num_objects":4770,"num_object_clones":0,"num_object_copies":14310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4770,"num_write_kb":305280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":470,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481645+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:55:18.481645+0000","last_peered":"2024-09-13T14:55:18.481645+0000","last_clean":"2024-09-13T14:55:18.481645+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:55:18.481645+0000","last_undegraded":"2024-09-13T14:55:18.481645+0000","last_fullsized":"2024-09-13T14:55:18.481645+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480598+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:55:18.480598+0000","last_peered":"2024-09-13T14:55:18.480598+0000","last_clean":"2024-09-13T14:55:18.480598+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:55:18.480598+0000","last_undegraded":"2024-09-13T14:55:18.480598+0000","last_fullsized":"2024-09-13T14:55:18.480598+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"459'4684","reported_seq":4715,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.167428+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:55:22.167428+0000","last_peered":"2024-09-13T14:55:22.167428+0000","last_clean":"2024-09-13T14:55:22.167428+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:55:22.167428+0000","last_undegraded":"2024-09-13T14:55:22.167428+0000","last_fullsized":"2024-09-13T14:55:22.167428+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4684,"log_dups_size":0,"ondisk_log_size":4684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306970624,"num_objects":4684,"num_object_clones":0,"num_object_copies":14052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4684,"num_write_kb":299776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481327+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:55:18.481327+0000","last_peered":"2024-09-13T14:55:18.481327+0000","last_clean":"2024-09-13T14:55:18.481327+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:55:18.481327+0000","last_undegraded":"2024-09-13T14:55:18.481327+0000","last_fullsized":"2024-09-13T14:55:18.481327+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":439,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485685+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:55:18.485685+0000","last_peered":"2024-09-13T14:55:18.485685+0000","last_clean":"2024-09-13T14:55:18.485685+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:55:18.485685+0000","last_undegraded":"2024-09-13T14:55:18.485685+0000","last_fullsized":"2024-09-13T14:55:18.485685+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"459'4578","reported_seq":4609,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.203437+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:55:22.203437+0000","last_peered":"2024-09-13T14:55:22.203437+0000","last_clean":"2024-09-13T14:55:22.203437+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:55:22.203437+0000","last_undegraded":"2024-09-13T14:55:22.203437+0000","last_fullsized":"2024-09-13T14:55:22.203437+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4578,"log_dups_size":0,"ondisk_log_size":4578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4578,"num_write_kb":292992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":387,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477980+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:55:18.477980+0000","last_peered":"2024-09-13T14:55:18.477980+0000","last_clean":"2024-09-13T14:55:18.477980+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:55:18.477980+0000","last_undegraded":"2024-09-13T14:55:18.477980+0000","last_fullsized":"2024-09-13T14:55:18.477980+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515397+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:55:19.515397+0000","last_peered":"2024-09-13T14:55:19.515397+0000","last_clean":"2024-09-13T14:55:19.515397+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:55:19.515397+0000","last_undegraded":"2024-09-13T14:55:19.515397+0000","last_fullsized":"2024-09-13T14:55:19.515397+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"459'4553","reported_seq":4584,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:20.678352+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:55:20.678352+0000","last_peered":"2024-09-13T14:55:20.678352+0000","last_clean":"2024-09-13T14:55:20.678352+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:55:20.678352+0000","last_undegraded":"2024-09-13T14:55:20.678352+0000","last_fullsized":"2024-09-13T14:55:20.678352+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298385408,"num_objects":4553,"num_object_clones":0,"num_object_copies":13659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4553,"num_write_kb":291392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":570,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.483837+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:55:18.483837+0000","last_peered":"2024-09-13T14:55:18.483837+0000","last_clean":"2024-09-13T14:55:18.483837+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:55:18.483837+0000","last_undegraded":"2024-09-13T14:55:18.483837+0000","last_fullsized":"2024-09-13T14:55:18.483837+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":835,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490904+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:55:18.490904+0000","last_peered":"2024-09-13T14:55:18.490904+0000","last_clean":"2024-09-13T14:55:18.490904+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:55:18.490904+0000","last_undegraded":"2024-09-13T14:55:18.490904+0000","last_fullsized":"2024-09-13T14:55:18.490904+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482125+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:55:18.482125+0000","last_peered":"2024-09-13T14:55:18.482125+0000","last_clean":"2024-09-13T14:55:18.482125+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:55:18.482125+0000","last_undegraded":"2024-09-13T14:55:18.482125+0000","last_fullsized":"2024-09-13T14:55:18.482125+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"459'4667","reported_seq":4685,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:23.287993+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:55:23.287993+0000","last_peered":"2024-09-13T14:55:23.287993+0000","last_clean":"2024-09-13T14:55:23.287993+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:55:23.287993+0000","last_undegraded":"2024-09-13T14:55:23.287993+0000","last_fullsized":"2024-09-13T14:55:23.287993+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4667,"log_dups_size":0,"ondisk_log_size":4667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305856512,"num_objects":4667,"num_object_clones":0,"num_object_copies":14001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4667,"num_write_kb":298688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":284,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477630+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:55:18.477630+0000","last_peered":"2024-09-13T14:55:18.477630+0000","last_clean":"2024-09-13T14:55:18.477630+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:55:18.477630+0000","last_undegraded":"2024-09-13T14:55:18.477630+0000","last_fullsized":"2024-09-13T14:55:18.477630+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":460,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485552+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:55:18.485552+0000","last_peered":"2024-09-13T14:55:18.485552+0000","last_clean":"2024-09-13T14:55:18.485552+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:55:18.485552+0000","last_undegraded":"2024-09-13T14:55:18.485552+0000","last_fullsized":"2024-09-13T14:55:18.485552+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481977+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:55:18.481977+0000","last_peered":"2024-09-13T14:55:18.481977+0000","last_clean":"2024-09-13T14:55:18.481977+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:55:18.481977+0000","last_undegraded":"2024-09-13T14:55:18.481977+0000","last_fullsized":"2024-09-13T14:55:18.481977+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"459'4614","reported_seq":4632,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.792918+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:55:22.792918+0000","last_peered":"2024-09-13T14:55:22.792918+0000","last_clean":"2024-09-13T14:55:22.792918+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:55:22.792918+0000","last_undegraded":"2024-09-13T14:55:22.792918+0000","last_fullsized":"2024-09-13T14:55:22.792918+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4614,"log_dups_size":0,"ondisk_log_size":4614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302383104,"num_objects":4614,"num_object_clones":0,"num_object_copies":13842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4614,"num_write_kb":295296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480373+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:55:18.480373+0000","last_peered":"2024-09-13T14:55:18.480373+0000","last_clean":"2024-09-13T14:55:18.480373+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:55:18.480373+0000","last_undegraded":"2024-09-13T14:55:18.480373+0000","last_fullsized":"2024-09-13T14:55:18.480373+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481220+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:55:18.481220+0000","last_peered":"2024-09-13T14:55:18.481220+0000","last_clean":"2024-09-13T14:55:18.481220+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:55:18.481220+0000","last_undegraded":"2024-09-13T14:55:18.481220+0000","last_fullsized":"2024-09-13T14:55:18.481220+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"459'4609","reported_seq":4627,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.896857+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:55:22.896857+0000","last_peered":"2024-09-13T14:55:22.896857+0000","last_clean":"2024-09-13T14:55:22.896857+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:55:22.896857+0000","last_undegraded":"2024-09-13T14:55:22.896857+0000","last_fullsized":"2024-09-13T14:55:22.896857+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4609,"log_dups_size":0,"ondisk_log_size":4609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302055424,"num_objects":4609,"num_object_clones":0,"num_object_copies":13827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4609,"num_write_kb":294976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":277,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477828+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:55:18.477828+0000","last_peered":"2024-09-13T14:55:18.477828+0000","last_clean":"2024-09-13T14:55:18.477828+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:55:18.477828+0000","last_undegraded":"2024-09-13T14:55:18.477828+0000","last_fullsized":"2024-09-13T14:55:18.477828+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":806,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490461+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:55:18.490461+0000","last_peered":"2024-09-13T14:55:18.490461+0000","last_clean":"2024-09-13T14:55:18.490461+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:55:18.490461+0000","last_undegraded":"2024-09-13T14:55:18.490461+0000","last_fullsized":"2024-09-13T14:55:18.490461+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":770,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522020+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:55:19.522020+0000","last_peered":"2024-09-13T14:55:19.522020+0000","last_clean":"2024-09-13T14:55:19.522020+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:55:19.522020+0000","last_undegraded":"2024-09-13T14:55:19.522020+0000","last_fullsized":"2024-09-13T14:55:19.522020+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"459'4664","reported_seq":4682,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.794497+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:55:22.794497+0000","last_peered":"2024-09-13T14:55:22.794497+0000","last_clean":"2024-09-13T14:55:22.794497+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:55:22.794497+0000","last_undegraded":"2024-09-13T14:55:22.794497+0000","last_fullsized":"2024-09-13T14:55:22.794497+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4664,"log_dups_size":0,"ondisk_log_size":4664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4664,"num_write_kb":298496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":581,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481215+0000","last_change":"2024-09-13T14:53:08.772753+0000","last_active":"2024-09-13T14:55:18.481215+0000","last_peered":"2024-09-13T14:55:18.481215+0000","last_clean":"2024-09-13T14:55:18.481215+0000","last_became_active":"2024-09-13T14:53:08.772283+0000","last_became_peered":"2024-09-13T14:53:08.772283+0000","last_unstale":"2024-09-13T14:55:18.481215+0000","last_undegraded":"2024-09-13T14:55:18.481215+0000","last_fullsized":"2024-09-13T14:55:18.481215+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:30.366156+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":983,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478521+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:55:18.478521+0000","last_peered":"2024-09-13T14:55:18.478521+0000","last_clean":"2024-09-13T14:55:18.478521+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:55:18.478521+0000","last_undegraded":"2024-09-13T14:55:18.478521+0000","last_fullsized":"2024-09-13T14:55:18.478521+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":760,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490364+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:55:18.490364+0000","last_peered":"2024-09-13T14:55:18.490364+0000","last_clean":"2024-09-13T14:55:18.490364+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:55:18.490364+0000","last_undegraded":"2024-09-13T14:55:18.490364+0000","last_fullsized":"2024-09-13T14:55:18.490364+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":425,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486082+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:55:18.486082+0000","last_peered":"2024-09-13T14:55:18.486082+0000","last_clean":"2024-09-13T14:55:18.486082+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:55:18.486082+0000","last_undegraded":"2024-09-13T14:55:18.486082+0000","last_fullsized":"2024-09-13T14:55:18.486082+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"459'4651","reported_seq":4682,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.174056+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:55:22.174056+0000","last_peered":"2024-09-13T14:55:22.174056+0000","last_clean":"2024-09-13T14:55:22.174056+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:55:22.174056+0000","last_undegraded":"2024-09-13T14:55:22.174056+0000","last_fullsized":"2024-09-13T14:55:22.174056+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304807936,"num_objects":4651,"num_object_clones":0,"num_object_copies":13953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4651,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":616,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.520497+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:55:19.520497+0000","last_peered":"2024-09-13T14:55:19.520497+0000","last_clean":"2024-09-13T14:55:19.520497+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:55:19.520497+0000","last_undegraded":"2024-09-13T14:55:19.520497+0000","last_fullsized":"2024-09-13T14:55:19.520497+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:32.116230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":871,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481139+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:55:18.481139+0000","last_peered":"2024-09-13T14:55:18.481139+0000","last_clean":"2024-09-13T14:55:18.481139+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:55:18.481139+0000","last_undegraded":"2024-09-13T14:55:18.481139+0000","last_fullsized":"2024-09-13T14:55:18.481139+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.514999+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:55:19.514999+0000","last_peered":"2024-09-13T14:55:19.514999+0000","last_clean":"2024-09-13T14:55:19.514999+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:55:19.514999+0000","last_undegraded":"2024-09-13T14:55:19.514999+0000","last_fullsized":"2024-09-13T14:55:19.514999+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515685+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:55:19.515685+0000","last_peered":"2024-09-13T14:55:19.515685+0000","last_clean":"2024-09-13T14:55:19.515685+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:55:19.515685+0000","last_undegraded":"2024-09-13T14:55:19.515685+0000","last_fullsized":"2024-09-13T14:55:19.515685+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"459'4752","reported_seq":4770,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:23.335476+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:55:23.335476+0000","last_peered":"2024-09-13T14:55:23.335476+0000","last_clean":"2024-09-13T14:55:23.335476+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:55:23.335476+0000","last_undegraded":"2024-09-13T14:55:23.335476+0000","last_fullsized":"2024-09-13T14:55:23.335476+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4752,"log_dups_size":0,"ondisk_log_size":4752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311427072,"num_objects":4752,"num_object_clones":0,"num_object_copies":14256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4752,"num_write_kb":304128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":610,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480673+0000","last_change":"2024-09-13T14:53:08.772285+0000","last_active":"2024-09-13T14:55:18.480673+0000","last_peered":"2024-09-13T14:55:18.480673+0000","last_clean":"2024-09-13T14:55:18.480673+0000","last_became_active":"2024-09-13T14:53:08.770613+0000","last_became_peered":"2024-09-13T14:53:08.770613+0000","last_unstale":"2024-09-13T14:55:18.480673+0000","last_undegraded":"2024-09-13T14:55:18.480673+0000","last_fullsized":"2024-09-13T14:55:18.480673+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:08.707450+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":786,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490675+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:55:18.490675+0000","last_peered":"2024-09-13T14:55:18.490675+0000","last_clean":"2024-09-13T14:55:18.490675+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:55:18.490675+0000","last_undegraded":"2024-09-13T14:55:18.490675+0000","last_fullsized":"2024-09-13T14:55:18.490675+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":748,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519518+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:55:19.519518+0000","last_peered":"2024-09-13T14:55:19.519518+0000","last_clean":"2024-09-13T14:55:19.519518+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:55:19.519518+0000","last_undegraded":"2024-09-13T14:55:19.519518+0000","last_fullsized":"2024-09-13T14:55:19.519518+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478156+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:55:18.478156+0000","last_peered":"2024-09-13T14:55:18.478156+0000","last_clean":"2024-09-13T14:55:18.478156+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:55:18.478156+0000","last_undegraded":"2024-09-13T14:55:18.478156+0000","last_fullsized":"2024-09-13T14:55:18.478156+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"459'4717","reported_seq":4735,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.804115+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:55:22.804115+0000","last_peered":"2024-09-13T14:55:22.804115+0000","last_clean":"2024-09-13T14:55:22.804115+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:55:22.804115+0000","last_undegraded":"2024-09-13T14:55:22.804115+0000","last_fullsized":"2024-09-13T14:55:22.804115+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4717,"log_dups_size":0,"ondisk_log_size":4717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309133312,"num_objects":4717,"num_object_clones":0,"num_object_copies":14151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4717,"num_write_kb":301888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":297,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481887+0000","last_change":"2024-09-13T14:53:08.772053+0000","last_active":"2024-09-13T14:55:18.481887+0000","last_peered":"2024-09-13T14:55:18.481887+0000","last_clean":"2024-09-13T14:55:18.481887+0000","last_became_active":"2024-09-13T14:53:08.769631+0000","last_became_peered":"2024-09-13T14:53:08.769631+0000","last_unstale":"2024-09-13T14:55:18.481887+0000","last_undegraded":"2024-09-13T14:55:18.481887+0000","last_fullsized":"2024-09-13T14:55:18.481887+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:53.493391+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1074,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477971+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:55:18.477971+0000","last_peered":"2024-09-13T14:55:18.477971+0000","last_clean":"2024-09-13T14:55:18.477971+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:55:18.477971+0000","last_undegraded":"2024-09-13T14:55:18.477971+0000","last_fullsized":"2024-09-13T14:55:18.477971+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482472+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:55:18.482472+0000","last_peered":"2024-09-13T14:55:18.482472+0000","last_clean":"2024-09-13T14:55:18.482472+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:55:18.482472+0000","last_undegraded":"2024-09-13T14:55:18.482472+0000","last_fullsized":"2024-09-13T14:55:18.482472+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":768,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.520450+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:55:19.520450+0000","last_peered":"2024-09-13T14:55:19.520450+0000","last_clean":"2024-09-13T14:55:19.520450+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:55:19.520450+0000","last_undegraded":"2024-09-13T14:55:19.520450+0000","last_fullsized":"2024-09-13T14:55:19.520450+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"459'4701","reported_seq":4719,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:23.223694+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:55:23.223694+0000","last_peered":"2024-09-13T14:55:23.223694+0000","last_clean":"2024-09-13T14:55:23.223694+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:55:23.223694+0000","last_undegraded":"2024-09-13T14:55:23.223694+0000","last_fullsized":"2024-09-13T14:55:23.223694+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4701,"num_write_kb":300864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486341+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:55:18.486341+0000","last_peered":"2024-09-13T14:55:18.486341+0000","last_clean":"2024-09-13T14:55:18.486341+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:55:18.486341+0000","last_undegraded":"2024-09-13T14:55:18.486341+0000","last_fullsized":"2024-09-13T14:55:18.486341+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":404,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478257+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:55:18.478257+0000","last_peered":"2024-09-13T14:55:18.478257+0000","last_clean":"2024-09-13T14:55:18.478257+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:55:18.478257+0000","last_undegraded":"2024-09-13T14:55:18.478257+0000","last_fullsized":"2024-09-13T14:55:18.478257+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1431,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522229+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:55:19.522229+0000","last_peered":"2024-09-13T14:55:19.522229+0000","last_clean":"2024-09-13T14:55:19.522229+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:55:19.522229+0000","last_undegraded":"2024-09-13T14:55:19.522229+0000","last_fullsized":"2024-09-13T14:55:19.522229+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":749,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519744+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:55:19.519744+0000","last_peered":"2024-09-13T14:55:19.519744+0000","last_clean":"2024-09-13T14:55:19.519744+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:55:19.519744+0000","last_undegraded":"2024-09-13T14:55:19.519744+0000","last_fullsized":"2024-09-13T14:55:19.519744+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"459'4629","reported_seq":4644,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.759496+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:55:22.759496+0000","last_peered":"2024-09-13T14:55:22.759496+0000","last_clean":"2024-09-13T14:55:22.759496+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:55:22.759496+0000","last_undegraded":"2024-09-13T14:55:22.759496+0000","last_fullsized":"2024-09-13T14:55:22.759496+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4629,"log_dups_size":0,"ondisk_log_size":4629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303366144,"num_objects":4629,"num_object_clones":0,"num_object_copies":13887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4629,"num_write_kb":296256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":478,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485875+0000","last_change":"2024-09-13T14:53:08.775380+0000","last_active":"2024-09-13T14:55:18.485875+0000","last_peered":"2024-09-13T14:55:18.485875+0000","last_clean":"2024-09-13T14:55:18.485875+0000","last_became_active":"2024-09-13T14:53:08.775030+0000","last_became_peered":"2024-09-13T14:53:08.775030+0000","last_unstale":"2024-09-13T14:55:18.485875+0000","last_undegraded":"2024-09-13T14:55:18.485875+0000","last_fullsized":"2024-09-13T14:55:18.485875+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:57.680954+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":758,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522139+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:55:19.522139+0000","last_peered":"2024-09-13T14:55:19.522139+0000","last_clean":"2024-09-13T14:55:19.522139+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:55:19.522139+0000","last_undegraded":"2024-09-13T14:55:19.522139+0000","last_fullsized":"2024-09-13T14:55:19.522139+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":875,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481404+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:55:18.481404+0000","last_peered":"2024-09-13T14:55:18.481404+0000","last_clean":"2024-09-13T14:55:18.481404+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:55:18.481404+0000","last_undegraded":"2024-09-13T14:55:18.481404+0000","last_fullsized":"2024-09-13T14:55:18.481404+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515153+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:55:19.515153+0000","last_peered":"2024-09-13T14:55:19.515153+0000","last_clean":"2024-09-13T14:55:19.515153+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:55:19.515153+0000","last_undegraded":"2024-09-13T14:55:19.515153+0000","last_fullsized":"2024-09-13T14:55:19.515153+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"459'4694","reported_seq":4711,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.900211+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:55:22.900211+0000","last_peered":"2024-09-13T14:55:22.900211+0000","last_clean":"2024-09-13T14:55:22.900211+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:55:22.900211+0000","last_undegraded":"2024-09-13T14:55:22.900211+0000","last_fullsized":"2024-09-13T14:55:22.900211+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4694,"log_dups_size":0,"ondisk_log_size":4694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307625984,"num_objects":4694,"num_object_clones":0,"num_object_copies":14082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4694,"num_write_kb":300416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481777+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:55:18.481777+0000","last_peered":"2024-09-13T14:55:18.481777+0000","last_clean":"2024-09-13T14:55:18.481777+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:55:18.481777+0000","last_undegraded":"2024-09-13T14:55:18.481777+0000","last_fullsized":"2024-09-13T14:55:18.481777+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":814,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.492167+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:55:18.492167+0000","last_peered":"2024-09-13T14:55:18.492167+0000","last_clean":"2024-09-13T14:55:18.492167+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:55:18.492167+0000","last_undegraded":"2024-09-13T14:55:18.492167+0000","last_fullsized":"2024-09-13T14:55:18.492167+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1134,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521764+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:55:19.521764+0000","last_peered":"2024-09-13T14:55:19.521764+0000","last_clean":"2024-09-13T14:55:19.521764+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:55:19.521764+0000","last_undegraded":"2024-09-13T14:55:19.521764+0000","last_fullsized":"2024-09-13T14:55:19.521764+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":804,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490261+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:55:18.490261+0000","last_peered":"2024-09-13T14:55:18.490261+0000","last_clean":"2024-09-13T14:55:18.490261+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:55:18.490261+0000","last_undegraded":"2024-09-13T14:55:18.490261+0000","last_fullsized":"2024-09-13T14:55:18.490261+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"459'4681","reported_seq":4699,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:22.789487+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:55:22.789487+0000","last_peered":"2024-09-13T14:55:22.789487+0000","last_clean":"2024-09-13T14:55:22.789487+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:55:22.789487+0000","last_undegraded":"2024-09-13T14:55:22.789487+0000","last_fullsized":"2024-09-13T14:55:22.789487+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":4681,"log_dups_size":0,"ondisk_log_size":4681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306774016,"num_objects":4681,"num_object_clones":0,"num_object_copies":14043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4681,"num_write_kb":299584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":645,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.491668+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:55:18.491668+0000","last_peered":"2024-09-13T14:55:18.491668+0000","last_clean":"2024-09-13T14:55:18.491668+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:55:18.491668+0000","last_undegraded":"2024-09-13T14:55:18.491668+0000","last_fullsized":"2024-09-13T14:55:18.491668+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:04:19.836641+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":97,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515742+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:55:19.515742+0000","last_peered":"2024-09-13T14:55:19.515742+0000","last_clean":"2024-09-13T14:55:19.515742+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:55:19.515742+0000","last_undegraded":"2024-09-13T14:55:19.515742+0000","last_fullsized":"2024-09-13T14:55:19.515742+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":877,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486505+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:55:18.486505+0000","last_peered":"2024-09-13T14:55:18.486505+0000","last_clean":"2024-09-13T14:55:18.486505+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:55:18.486505+0000","last_undegraded":"2024-09-13T14:55:18.486505+0000","last_fullsized":"2024-09-13T14:55:18.486505+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480489+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:55:18.480489+0000","last_peered":"2024-09-13T14:55:18.480489+0000","last_clean":"2024-09-13T14:55:18.480489+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:55:18.480489+0000","last_undegraded":"2024-09-13T14:55:18.480489+0000","last_fullsized":"2024-09-13T14:55:18.480489+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482533+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:55:18.482533+0000","last_peered":"2024-09-13T14:55:18.482533+0000","last_clean":"2024-09-13T14:55:18.482533+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:55:18.482533+0000","last_undegraded":"2024-09-13T14:55:18.482533+0000","last_fullsized":"2024-09-13T14:55:18.482533+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":167,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482096+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:55:18.482096+0000","last_peered":"2024-09-13T14:55:18.482096+0000","last_clean":"2024-09-13T14:55:18.482096+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:55:18.482096+0000","last_undegraded":"2024-09-13T14:55:18.482096+0000","last_fullsized":"2024-09-13T14:55:18.482096+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":745,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519621+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:55:19.519621+0000","last_peered":"2024-09-13T14:55:19.519621+0000","last_clean":"2024-09-13T14:55:19.519621+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:55:19.519621+0000","last_undegraded":"2024-09-13T14:55:19.519621+0000","last_fullsized":"2024-09-13T14:55:19.519621+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481200+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:55:18.481200+0000","last_peered":"2024-09-13T14:55:18.481200+0000","last_clean":"2024-09-13T14:55:18.481200+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:55:18.481200+0000","last_undegraded":"2024-09-13T14:55:18.481200+0000","last_fullsized":"2024-09-13T14:55:18.481200+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477993+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:55:18.477993+0000","last_peered":"2024-09-13T14:55:18.477993+0000","last_clean":"2024-09-13T14:55:18.477993+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:55:18.477993+0000","last_undegraded":"2024-09-13T14:55:18.477993+0000","last_fullsized":"2024-09-13T14:55:18.477993+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":748,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490153+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:55:18.490153+0000","last_peered":"2024-09-13T14:55:18.490153+0000","last_clean":"2024-09-13T14:55:18.490153+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:55:18.490153+0000","last_undegraded":"2024-09-13T14:55:18.490153+0000","last_fullsized":"2024-09-13T14:55:18.490153+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515906+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:55:19.515906+0000","last_peered":"2024-09-13T14:55:19.515906+0000","last_clean":"2024-09-13T14:55:19.515906+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:55:19.515906+0000","last_undegraded":"2024-09-13T14:55:19.515906+0000","last_fullsized":"2024-09-13T14:55:19.515906+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":804,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490165+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:55:18.490165+0000","last_peered":"2024-09-13T14:55:18.490165+0000","last_clean":"2024-09-13T14:55:18.490165+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:55:18.490165+0000","last_undegraded":"2024-09-13T14:55:18.490165+0000","last_fullsized":"2024-09-13T14:55:18.490165+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":456,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486410+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:55:18.486410+0000","last_peered":"2024-09-13T14:55:18.486410+0000","last_clean":"2024-09-13T14:55:18.486410+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:55:18.486410+0000","last_undegraded":"2024-09-13T14:55:18.486410+0000","last_fullsized":"2024-09-13T14:55:18.486410+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":835,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490351+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:55:18.490351+0000","last_peered":"2024-09-13T14:55:18.490351+0000","last_clean":"2024-09-13T14:55:18.490351+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:55:18.490351+0000","last_undegraded":"2024-09-13T14:55:18.490351+0000","last_fullsized":"2024-09-13T14:55:18.490351+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515811+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:55:19.515811+0000","last_peered":"2024-09-13T14:55:19.515811+0000","last_clean":"2024-09-13T14:55:19.515811+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:55:19.515811+0000","last_undegraded":"2024-09-13T14:55:19.515811+0000","last_fullsized":"2024-09-13T14:55:19.515811+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":770,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519374+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:55:19.519374+0000","last_peered":"2024-09-13T14:55:19.519374+0000","last_clean":"2024-09-13T14:55:19.519374+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:55:19.519374+0000","last_undegraded":"2024-09-13T14:55:19.519374+0000","last_fullsized":"2024-09-13T14:55:19.519374+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":817,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.491045+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:55:18.491045+0000","last_peered":"2024-09-13T14:55:18.491045+0000","last_clean":"2024-09-13T14:55:18.491045+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:55:18.491045+0000","last_undegraded":"2024-09-13T14:55:18.491045+0000","last_fullsized":"2024-09-13T14:55:18.491045+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":156,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480478+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:55:18.480478+0000","last_peered":"2024-09-13T14:55:18.480478+0000","last_clean":"2024-09-13T14:55:18.480478+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:55:18.480478+0000","last_undegraded":"2024-09-13T14:55:18.480478+0000","last_fullsized":"2024-09-13T14:55:18.480478+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":716,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522362+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:55:19.522362+0000","last_peered":"2024-09-13T14:55:19.522362+0000","last_clean":"2024-09-13T14:55:19.522362+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:55:19.522362+0000","last_undegraded":"2024-09-13T14:55:19.522362+0000","last_fullsized":"2024-09-13T14:55:19.522362+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515244+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:55:19.515244+0000","last_peered":"2024-09-13T14:55:19.515244+0000","last_clean":"2024-09-13T14:55:19.515244+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:55:19.515244+0000","last_undegraded":"2024-09-13T14:55:19.515244+0000","last_fullsized":"2024-09-13T14:55:19.515244+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482408+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:55:18.482408+0000","last_peered":"2024-09-13T14:55:18.482408+0000","last_clean":"2024-09-13T14:55:18.482408+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:55:18.482408+0000","last_undegraded":"2024-09-13T14:55:18.482408+0000","last_fullsized":"2024-09-13T14:55:18.482408+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478071+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:55:18.478071+0000","last_peered":"2024-09-13T14:55:18.478071+0000","last_clean":"2024-09-13T14:55:18.478071+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:55:18.478071+0000","last_undegraded":"2024-09-13T14:55:18.478071+0000","last_fullsized":"2024-09-13T14:55:18.478071+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480554+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:55:18.480554+0000","last_peered":"2024-09-13T14:55:18.480554+0000","last_clean":"2024-09-13T14:55:18.480554+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:55:18.480554+0000","last_undegraded":"2024-09-13T14:55:18.480554+0000","last_fullsized":"2024-09-13T14:55:18.480554+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":390,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478429+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:55:18.478429+0000","last_peered":"2024-09-13T14:55:18.478429+0000","last_clean":"2024-09-13T14:55:18.478429+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:55:18.478429+0000","last_undegraded":"2024-09-13T14:55:18.478429+0000","last_fullsized":"2024-09-13T14:55:18.478429+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":827,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.490774+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:55:18.490774+0000","last_peered":"2024-09-13T14:55:18.490774+0000","last_clean":"2024-09-13T14:55:18.490774+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:55:18.490774+0000","last_undegraded":"2024-09-13T14:55:18.490774+0000","last_fullsized":"2024-09-13T14:55:18.490774+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":801,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.491440+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:55:18.491440+0000","last_peered":"2024-09-13T14:55:18.491440+0000","last_clean":"2024-09-13T14:55:18.491440+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:55:18.491440+0000","last_undegraded":"2024-09-13T14:55:18.491440+0000","last_fullsized":"2024-09-13T14:55:18.491440+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482219+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:55:18.482219+0000","last_peered":"2024-09-13T14:55:18.482219+0000","last_clean":"2024-09-13T14:55:18.482219+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:55:18.482219+0000","last_undegraded":"2024-09-13T14:55:18.482219+0000","last_fullsized":"2024-09-13T14:55:18.482219+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":99,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515789+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:55:19.515789+0000","last_peered":"2024-09-13T14:55:19.515789+0000","last_clean":"2024-09-13T14:55:19.515789+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:55:19.515789+0000","last_undegraded":"2024-09-13T14:55:19.515789+0000","last_fullsized":"2024-09-13T14:55:19.515789+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4891475968,"num_objects":74638,"num_object_clones":0,"num_object_copies":223914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74638,"num_write_kb":4776832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14486798336,"data_stored":14486798336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74638,"ondisk_log_size":74638,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398031,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089400,"kb_used_data":1197552,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91655752,"statfs":{"total":95995035648,"available":93855490048,"internally_reserved":0,"allocated":1226293248,"data_stored":1223848758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594259,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2995048,"kb_used_data":2100576,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":90750104,"statfs":{"total":95995035648,"available":92928106496,"internally_reserved":0,"allocated":2150989824,"data_stored":2148561016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665713,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2442084,"kb_used_data":1520220,"kb_used_omap":9,"kb_used_meta":921846,"kb_avail":91303068,"statfs":{"total":95995035648,"available":93494341632,"internally_reserved":0,"allocated":1556705280,"data_stored":1554276875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":943970504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986715,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3275444,"kb_used_data":2383916,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":90469708,"statfs":{"total":95995035648,"available":92640980992,"internally_reserved":0,"allocated":2441129984,"data_stored":2438689520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697274,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2479180,"kb_used_data":1452164,"kb_used_omap":4,"kb_used_meta":1027003,"kb_avail":91265972,"statfs":{"total":95995035648,"available":93456355328,"internally_reserved":0,"allocated":1487015936,"data_stored":1484595512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1051651147},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860801,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2530864,"kb_used_data":1729248,"kb_used_omap":8,"kb_used_meta":801591,"kb_avail":91214288,"statfs":{"total":95995035648,"available":93403430912,"internally_reserved":0,"allocated":1770749952,"data_stored":1768318046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":820830028},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020472,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470180,"kb_used_data":867800,"kb_used_omap":9,"kb_used_meta":602358,"kb_avail":92274972,"statfs":{"total":95995035648,"available":94489571328,"internally_reserved":0,"allocated":888627200,"data_stored":886202349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":616815429},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187863,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3813476,"kb_used_data":2950488,"kb_used_omap":7,"kb_used_meta":862968,"kb_avail":89931676,"statfs":{"total":95995035648,"available":92090036224,"internally_reserved":0,"allocated":3021299712,"data_stored":3018875127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":883679563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,2,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3029663744,"data_stored":3029663744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":886243328,"data_stored":886243328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1767112704,"data_stored":1767112704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1426980864,"data_stored":1426980864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2446000128,"data_stored":2446000128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1551171584,"data_stored":1551171584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2147155968,"data_stored":2147155968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1232470016,"data_stored":1232470016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:28.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:28.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:28.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:28.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:28.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[103001]: pgmap v1498: 99 pgs: 99 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:55:28.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:28.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:28.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:28 smithi017 ceph-mon[107556]: pgmap v1498: 99 pgs: 99 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:55:28.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:28.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:28 smithi189 ceph-mon[90992]: pgmap v1498: 99 pgs: 99 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2024-09-13T14:55:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:28.647 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:28.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:28.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:28.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:28.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:28.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 81985 81969 111.357 69.8125 0.0097245 0.00897555 2024-09-13T14:55:29.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:29.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:29.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.267 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.368 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:29.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:29.512 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.550 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.575 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:29.651 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.677 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:29.677 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:29.696 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.798 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:29.838 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:29.883 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:29.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:29.996 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 83208 83192 110.614 76.4375 0.0040679 0.00903453 2024-09-13T14:55:30.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:30.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:30.329 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:30.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:30.354 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:55:30.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:30.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:30.475 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:30.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:30.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.602 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:30.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:30.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.723 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:30 smithi189 ceph-mon[90992]: pgmap v1499: 99 pgs: 99 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:55:30.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:30.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:30.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:30.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:30 smithi017 ceph-mon[103001]: pgmap v1499: 99 pgs: 99 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:55:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:30 smithi017 ceph-mon[107556]: pgmap v1499: 99 pgs: 99 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:55:30.973 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:30.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 83648 83632 108.882 27.5 0.00734201 0.00917857 2024-09-13T14:55:31.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:31.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:31.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:31.250 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:31.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:31.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:31.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:31.499 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:31.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:31.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:31.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:31.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:31.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:31.898 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:31.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:31.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:31.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 84443 84427 107.674 49.6875 0.0207786 0.00928177 2024-09-13T14:55:32.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:32.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:32.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:32.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:32.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:32.499 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:32.589 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:32 smithi017 ceph-mon[107556]: pgmap v1500: 99 pgs: 99 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:32.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:32 smithi017 ceph-mon[103001]: pgmap v1500: 99 pgs: 99 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:32.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.657 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:32.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:32 smithi189 ceph-mon[90992]: pgmap v1500: 99 pgs: 99 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:32.758 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.790 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.892 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:32.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:32.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 85589 85573 106.953 71.625 0.0250356 0.00934402 2024-09-13T14:55:33.030 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:33.073 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:33.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:33.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:33.319 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:33.327 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:33.327 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:33.357 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:33.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:33.459 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.495 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:33.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:33.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:33.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:33.657 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:55:33.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/470071884' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/470071884' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/470071884' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:33.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:33.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 86806 86790 106.347 76.0625 0.00998729 0.00939803 2024-09-13T14:55:34.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:55:34.015 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.097 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:34.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:34.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:34.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:34.331 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.393 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.449 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:34.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:34.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:34.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:34.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:34.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:34 smithi189 ceph-mon[90992]: pgmap v1501: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:55:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:34 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:34.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:34 smithi189 ceph-mon[90992]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:55:34.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:34.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[103001]: pgmap v1501: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[103001]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[107556]: pgmap v1501: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:34.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:34 smithi017 ceph-mon[107556]: osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:55:34.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.963 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:34.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 87952 87936 105.679 71.625 0.00954209 0.00945709 2024-09-13T14:55:35.064 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:35.095 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:35.114 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:55:35.117 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:55:35.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:35.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:35.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:35.343 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:35.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:35.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:35.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:35.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.661 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2087416658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2087416658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:35.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2087416658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:35.764 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:35.793 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:35.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:35.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:35.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 89124 89108 105.067 73.25 0.0115858 0.00951233 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:55:36.029 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:36.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:36.034 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:36.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:36.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:36.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:36.313 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:36.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:36.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:36.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:36.503 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:36.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:36.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:36 smithi017 ceph-mon[103001]: from='client.47021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:36 smithi017 ceph-mon[103001]: pgmap v1503: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:55:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:36 smithi017 ceph-mon[107556]: from='client.47021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:36 smithi017 ceph-mon[107556]: pgmap v1503: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:55:36.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.752 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1502,"stamp":"2024-09-13T14:55:33.492749+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5596780493,"num_objects":85355,"num_object_clones":0,"num_object_copies":256065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85355,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":86234,"num_write_kb":5473551,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":8394258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87459,"ondisk_log_size":87459,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23185924,"kb_used_data":16245060,"kb_used_omap":44,"kb_used_meta":6940755,"kb_avail":726775292,"statfs":{"total":767960285184,"available":744217899008,"internally_reserved":0,"allocated":16634941440,"data_stored":16615489607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7107333367},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":700514304,"num_objects":10689,"num_object_clones":0,"num_object_copies":32067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10689,"num_write_kb":684096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.073271"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":806,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515214+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:55:19.515214+0000","last_peered":"2024-09-13T14:55:19.515214+0000","last_clean":"2024-09-13T14:55:19.515214+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:55:19.515214+0000","last_undegraded":"2024-09-13T14:55:19.515214+0000","last_fullsized":"2024-09-13T14:55:19.515214+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":457,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480988+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:55:18.480988+0000","last_peered":"2024-09-13T14:55:18.480988+0000","last_clean":"2024-09-13T14:55:18.480988+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:55:18.480988+0000","last_undegraded":"2024-09-13T14:55:18.480988+0000","last_fullsized":"2024-09-13T14:55:18.480988+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":386,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478359+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:55:18.478359+0000","last_peered":"2024-09-13T14:55:18.478359+0000","last_clean":"2024-09-13T14:55:18.478359+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:55:18.478359+0000","last_undegraded":"2024-09-13T14:55:18.478359+0000","last_fullsized":"2024-09-13T14:55:18.478359+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515664+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:55:19.515664+0000","last_peered":"2024-09-13T14:55:19.515664+0000","last_clean":"2024-09-13T14:55:19.515664+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:55:19.515664+0000","last_undegraded":"2024-09-13T14:55:19.515664+0000","last_fullsized":"2024-09-13T14:55:19.515664+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515734+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:55:19.515734+0000","last_peered":"2024-09-13T14:55:19.515734+0000","last_clean":"2024-09-13T14:55:19.515734+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:55:19.515734+0000","last_undegraded":"2024-09-13T14:55:19.515734+0000","last_fullsized":"2024-09-13T14:55:19.515734+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":426,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.484045+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:55:18.484045+0000","last_peered":"2024-09-13T14:55:18.484045+0000","last_clean":"2024-09-13T14:55:18.484045+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:55:18.484045+0000","last_undegraded":"2024-09-13T14:55:18.484045+0000","last_fullsized":"2024-09-13T14:55:18.484045+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":806,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515405+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:55:19.515405+0000","last_peered":"2024-09-13T14:55:19.515405+0000","last_clean":"2024-09-13T14:55:19.515405+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:55:19.515405+0000","last_undegraded":"2024-09-13T14:55:19.515405+0000","last_fullsized":"2024-09-13T14:55:19.515405+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":488,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486399+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:55:18.486399+0000","last_peered":"2024-09-13T14:55:18.486399+0000","last_clean":"2024-09-13T14:55:18.486399+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:55:18.486399+0000","last_undegraded":"2024-09-13T14:55:18.486399+0000","last_fullsized":"2024-09-13T14:55:18.486399+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"459'5317","reported_seq":5334,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.341098+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:55:33.341098+0000","last_peered":"2024-09-13T14:55:33.341098+0000","last_clean":"2024-09-13T14:55:33.341098+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:55:33.341098+0000","last_undegraded":"2024-09-13T14:55:33.341098+0000","last_fullsized":"2024-09-13T14:55:33.341098+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5317,"log_dups_size":0,"ondisk_log_size":5317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348454912,"num_objects":5317,"num_object_clones":0,"num_object_copies":15951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5317,"num_write_kb":340288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":480,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481899+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:55:18.481899+0000","last_peered":"2024-09-13T14:55:18.481899+0000","last_clean":"2024-09-13T14:55:18.481899+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:55:18.481899+0000","last_undegraded":"2024-09-13T14:55:18.481899+0000","last_fullsized":"2024-09-13T14:55:18.481899+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":371,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477551+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:55:18.477551+0000","last_peered":"2024-09-13T14:55:18.477551+0000","last_clean":"2024-09-13T14:55:18.477551+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:55:18.477551+0000","last_undegraded":"2024-09-13T14:55:18.477551+0000","last_fullsized":"2024-09-13T14:55:18.477551+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"459'5484","reported_seq":5502,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.801496+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:55:32.801496+0000","last_peered":"2024-09-13T14:55:32.801496+0000","last_clean":"2024-09-13T14:55:32.801496+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:55:32.801496+0000","last_undegraded":"2024-09-13T14:55:32.801496+0000","last_fullsized":"2024-09-13T14:55:32.801496+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359399424,"num_objects":5484,"num_object_clones":0,"num_object_copies":16452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5484,"num_write_kb":350976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":470,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481645+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:55:18.481645+0000","last_peered":"2024-09-13T14:55:18.481645+0000","last_clean":"2024-09-13T14:55:18.481645+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:55:18.481645+0000","last_undegraded":"2024-09-13T14:55:18.481645+0000","last_fullsized":"2024-09-13T14:55:18.481645+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480598+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:55:18.480598+0000","last_peered":"2024-09-13T14:55:18.480598+0000","last_clean":"2024-09-13T14:55:18.480598+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:55:18.480598+0000","last_undegraded":"2024-09-13T14:55:18.480598+0000","last_fullsized":"2024-09-13T14:55:18.480598+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"459'5341","reported_seq":5372,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.212515+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:55:32.212515+0000","last_peered":"2024-09-13T14:55:32.212515+0000","last_clean":"2024-09-13T14:55:32.212515+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:55:32.212515+0000","last_undegraded":"2024-09-13T14:55:32.212515+0000","last_fullsized":"2024-09-13T14:55:32.212515+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350027776,"num_objects":5341,"num_object_clones":0,"num_object_copies":16023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5341,"num_write_kb":341824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481327+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:55:18.481327+0000","last_peered":"2024-09-13T14:55:18.481327+0000","last_clean":"2024-09-13T14:55:18.481327+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:55:18.481327+0000","last_undegraded":"2024-09-13T14:55:18.481327+0000","last_fullsized":"2024-09-13T14:55:18.481327+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":439,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485685+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:55:18.485685+0000","last_peered":"2024-09-13T14:55:18.485685+0000","last_clean":"2024-09-13T14:55:18.485685+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:55:18.485685+0000","last_undegraded":"2024-09-13T14:55:18.485685+0000","last_fullsized":"2024-09-13T14:55:18.485685+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"459'5250","reported_seq":5281,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.189400+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:55:32.189400+0000","last_peered":"2024-09-13T14:55:32.189400+0000","last_clean":"2024-09-13T14:55:32.189400+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:55:32.189400+0000","last_undegraded":"2024-09-13T14:55:32.189400+0000","last_fullsized":"2024-09-13T14:55:32.189400+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5250,"log_dups_size":0,"ondisk_log_size":5250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344064000,"num_objects":5250,"num_object_clones":0,"num_object_copies":15750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5250,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":387,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477980+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:55:18.477980+0000","last_peered":"2024-09-13T14:55:18.477980+0000","last_clean":"2024-09-13T14:55:18.477980+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:55:18.477980+0000","last_undegraded":"2024-09-13T14:55:18.477980+0000","last_fullsized":"2024-09-13T14:55:18.477980+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515432+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:55:19.515432+0000","last_peered":"2024-09-13T14:55:19.515432+0000","last_clean":"2024-09-13T14:55:19.515432+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:55:19.515432+0000","last_undegraded":"2024-09-13T14:55:19.515432+0000","last_fullsized":"2024-09-13T14:55:19.515432+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"459'5268","reported_seq":5299,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:30.676933+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:55:30.676933+0000","last_peered":"2024-09-13T14:55:30.676933+0000","last_clean":"2024-09-13T14:55:30.676933+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:55:30.676933+0000","last_undegraded":"2024-09-13T14:55:30.676933+0000","last_fullsized":"2024-09-13T14:55:30.676933+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5268,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":570,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.483837+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:55:18.483837+0000","last_peered":"2024-09-13T14:55:18.483837+0000","last_clean":"2024-09-13T14:55:18.483837+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:55:18.483837+0000","last_undegraded":"2024-09-13T14:55:18.483837+0000","last_fullsized":"2024-09-13T14:55:18.483837+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":838,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.514982+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:55:19.514982+0000","last_peered":"2024-09-13T14:55:19.514982+0000","last_clean":"2024-09-13T14:55:19.514982+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:55:19.514982+0000","last_undegraded":"2024-09-13T14:55:19.514982+0000","last_fullsized":"2024-09-13T14:55:19.514982+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482125+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:55:18.482125+0000","last_peered":"2024-09-13T14:55:18.482125+0000","last_clean":"2024-09-13T14:55:18.482125+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:55:18.482125+0000","last_undegraded":"2024-09-13T14:55:18.482125+0000","last_fullsized":"2024-09-13T14:55:18.482125+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"459'5309","reported_seq":5327,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.326045+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:55:33.326045+0000","last_peered":"2024-09-13T14:55:33.326045+0000","last_clean":"2024-09-13T14:55:33.326045+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:55:33.326045+0000","last_undegraded":"2024-09-13T14:55:33.326045+0000","last_fullsized":"2024-09-13T14:55:33.326045+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5309,"num_write_kb":339776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":284,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477630+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:55:18.477630+0000","last_peered":"2024-09-13T14:55:18.477630+0000","last_clean":"2024-09-13T14:55:18.477630+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:55:18.477630+0000","last_undegraded":"2024-09-13T14:55:18.477630+0000","last_fullsized":"2024-09-13T14:55:18.477630+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":460,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485552+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:55:18.485552+0000","last_peered":"2024-09-13T14:55:18.485552+0000","last_clean":"2024-09-13T14:55:18.485552+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:55:18.485552+0000","last_undegraded":"2024-09-13T14:55:18.485552+0000","last_fullsized":"2024-09-13T14:55:18.485552+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481977+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:55:18.481977+0000","last_peered":"2024-09-13T14:55:18.481977+0000","last_clean":"2024-09-13T14:55:18.481977+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:55:18.481977+0000","last_undegraded":"2024-09-13T14:55:18.481977+0000","last_fullsized":"2024-09-13T14:55:18.481977+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"459'5274","reported_seq":5292,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.794189+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:55:32.794189+0000","last_peered":"2024-09-13T14:55:32.794189+0000","last_clean":"2024-09-13T14:55:32.794189+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:55:32.794189+0000","last_undegraded":"2024-09-13T14:55:32.794189+0000","last_fullsized":"2024-09-13T14:55:32.794189+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345636864,"num_objects":5274,"num_object_clones":0,"num_object_copies":15822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5274,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480373+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:55:18.480373+0000","last_peered":"2024-09-13T14:55:18.480373+0000","last_clean":"2024-09-13T14:55:18.480373+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:55:18.480373+0000","last_undegraded":"2024-09-13T14:55:18.480373+0000","last_fullsized":"2024-09-13T14:55:18.480373+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481220+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:55:18.481220+0000","last_peered":"2024-09-13T14:55:18.481220+0000","last_clean":"2024-09-13T14:55:18.481220+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:55:18.481220+0000","last_undegraded":"2024-09-13T14:55:18.481220+0000","last_fullsized":"2024-09-13T14:55:18.481220+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"459'5236","reported_seq":5254,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.887401+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:55:32.887401+0000","last_peered":"2024-09-13T14:55:32.887401+0000","last_clean":"2024-09-13T14:55:32.887401+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:55:32.887401+0000","last_undegraded":"2024-09-13T14:55:32.887401+0000","last_fullsized":"2024-09-13T14:55:32.887401+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5236,"log_dups_size":0,"ondisk_log_size":5236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343146496,"num_objects":5236,"num_object_clones":0,"num_object_copies":15708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5236,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":277,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477828+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:55:18.477828+0000","last_peered":"2024-09-13T14:55:18.477828+0000","last_clean":"2024-09-13T14:55:18.477828+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:55:18.477828+0000","last_undegraded":"2024-09-13T14:55:18.477828+0000","last_fullsized":"2024-09-13T14:55:18.477828+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":808,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513025+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:55:19.513025+0000","last_peered":"2024-09-13T14:55:19.513025+0000","last_clean":"2024-09-13T14:55:19.513025+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:55:19.513025+0000","last_undegraded":"2024-09-13T14:55:19.513025+0000","last_fullsized":"2024-09-13T14:55:19.513025+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":770,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522020+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:55:19.522020+0000","last_peered":"2024-09-13T14:55:19.522020+0000","last_clean":"2024-09-13T14:55:19.522020+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:55:19.522020+0000","last_undegraded":"2024-09-13T14:55:19.522020+0000","last_fullsized":"2024-09-13T14:55:19.522020+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"459'5318","reported_seq":5336,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.811815+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:55:32.811815+0000","last_peered":"2024-09-13T14:55:32.811815+0000","last_clean":"2024-09-13T14:55:32.811815+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:55:32.811815+0000","last_undegraded":"2024-09-13T14:55:32.811815+0000","last_fullsized":"2024-09-13T14:55:32.811815+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348520448,"num_objects":5318,"num_object_clones":0,"num_object_copies":15954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5318,"num_write_kb":340352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":581,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481215+0000","last_change":"2024-09-13T14:53:08.772753+0000","last_active":"2024-09-13T14:55:18.481215+0000","last_peered":"2024-09-13T14:55:18.481215+0000","last_clean":"2024-09-13T14:55:18.481215+0000","last_became_active":"2024-09-13T14:53:08.772283+0000","last_became_peered":"2024-09-13T14:53:08.772283+0000","last_unstale":"2024-09-13T14:55:18.481215+0000","last_undegraded":"2024-09-13T14:55:18.481215+0000","last_fullsized":"2024-09-13T14:55:18.481215+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:30.366156+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":983,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478521+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:55:18.478521+0000","last_peered":"2024-09-13T14:55:18.478521+0000","last_clean":"2024-09-13T14:55:18.478521+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:55:18.478521+0000","last_undegraded":"2024-09-13T14:55:18.478521+0000","last_fullsized":"2024-09-13T14:55:18.478521+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":762,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513693+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:55:19.513693+0000","last_peered":"2024-09-13T14:55:19.513693+0000","last_clean":"2024-09-13T14:55:19.513693+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:55:19.513693+0000","last_undegraded":"2024-09-13T14:55:19.513693+0000","last_fullsized":"2024-09-13T14:55:19.513693+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":425,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486082+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:55:18.486082+0000","last_peered":"2024-09-13T14:55:18.486082+0000","last_clean":"2024-09-13T14:55:18.486082+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:55:18.486082+0000","last_undegraded":"2024-09-13T14:55:18.486082+0000","last_fullsized":"2024-09-13T14:55:18.486082+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"459'5331","reported_seq":5362,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.212836+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:55:32.212836+0000","last_peered":"2024-09-13T14:55:32.212836+0000","last_clean":"2024-09-13T14:55:32.212836+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:55:32.212836+0000","last_undegraded":"2024-09-13T14:55:32.212836+0000","last_fullsized":"2024-09-13T14:55:32.212836+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5331,"log_dups_size":0,"ondisk_log_size":5331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5331,"num_write_kb":341184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":616,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.520497+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:55:19.520497+0000","last_peered":"2024-09-13T14:55:19.520497+0000","last_clean":"2024-09-13T14:55:19.520497+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:55:19.520497+0000","last_undegraded":"2024-09-13T14:55:19.520497+0000","last_fullsized":"2024-09-13T14:55:19.520497+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:32.116230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":871,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481139+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:55:18.481139+0000","last_peered":"2024-09-13T14:55:18.481139+0000","last_clean":"2024-09-13T14:55:18.481139+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:55:18.481139+0000","last_undegraded":"2024-09-13T14:55:18.481139+0000","last_fullsized":"2024-09-13T14:55:18.481139+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.514999+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:55:19.514999+0000","last_peered":"2024-09-13T14:55:19.514999+0000","last_clean":"2024-09-13T14:55:19.514999+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:55:19.514999+0000","last_undegraded":"2024-09-13T14:55:19.514999+0000","last_fullsized":"2024-09-13T14:55:19.514999+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515685+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:55:19.515685+0000","last_peered":"2024-09-13T14:55:19.515685+0000","last_clean":"2024-09-13T14:55:19.515685+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:55:19.515685+0000","last_undegraded":"2024-09-13T14:55:19.515685+0000","last_fullsized":"2024-09-13T14:55:19.515685+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"459'5414","reported_seq":5432,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.300230+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:55:33.300230+0000","last_peered":"2024-09-13T14:55:33.300230+0000","last_clean":"2024-09-13T14:55:33.300230+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:55:33.300230+0000","last_undegraded":"2024-09-13T14:55:33.300230+0000","last_fullsized":"2024-09-13T14:55:33.300230+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5414,"log_dups_size":0,"ondisk_log_size":5414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354811904,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5414,"num_write_kb":346496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":610,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480673+0000","last_change":"2024-09-13T14:53:08.772285+0000","last_active":"2024-09-13T14:55:18.480673+0000","last_peered":"2024-09-13T14:55:18.480673+0000","last_clean":"2024-09-13T14:55:18.480673+0000","last_became_active":"2024-09-13T14:53:08.770613+0000","last_became_peered":"2024-09-13T14:53:08.770613+0000","last_unstale":"2024-09-13T14:55:18.480673+0000","last_undegraded":"2024-09-13T14:55:18.480673+0000","last_fullsized":"2024-09-13T14:55:18.480673+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:08.707450+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":788,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513448+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:55:19.513448+0000","last_peered":"2024-09-13T14:55:19.513448+0000","last_clean":"2024-09-13T14:55:19.513448+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:55:19.513448+0000","last_undegraded":"2024-09-13T14:55:19.513448+0000","last_fullsized":"2024-09-13T14:55:19.513448+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":748,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519518+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:55:19.519518+0000","last_peered":"2024-09-13T14:55:19.519518+0000","last_clean":"2024-09-13T14:55:19.519518+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:55:19.519518+0000","last_undegraded":"2024-09-13T14:55:19.519518+0000","last_fullsized":"2024-09-13T14:55:19.519518+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478156+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:55:18.478156+0000","last_peered":"2024-09-13T14:55:18.478156+0000","last_clean":"2024-09-13T14:55:18.478156+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:55:18.478156+0000","last_undegraded":"2024-09-13T14:55:18.478156+0000","last_fullsized":"2024-09-13T14:55:18.478156+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"459'5390","reported_seq":5408,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.809431+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:55:32.809431+0000","last_peered":"2024-09-13T14:55:32.809431+0000","last_clean":"2024-09-13T14:55:32.809431+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:55:32.809431+0000","last_undegraded":"2024-09-13T14:55:32.809431+0000","last_fullsized":"2024-09-13T14:55:32.809431+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353239040,"num_objects":5390,"num_object_clones":0,"num_object_copies":16170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5390,"num_write_kb":344960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":297,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481887+0000","last_change":"2024-09-13T14:53:08.772053+0000","last_active":"2024-09-13T14:55:18.481887+0000","last_peered":"2024-09-13T14:55:18.481887+0000","last_clean":"2024-09-13T14:55:18.481887+0000","last_became_active":"2024-09-13T14:53:08.769631+0000","last_became_peered":"2024-09-13T14:53:08.769631+0000","last_unstale":"2024-09-13T14:55:18.481887+0000","last_undegraded":"2024-09-13T14:55:18.481887+0000","last_fullsized":"2024-09-13T14:55:18.481887+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:53.493391+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1074,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477971+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:55:18.477971+0000","last_peered":"2024-09-13T14:55:18.477971+0000","last_clean":"2024-09-13T14:55:18.477971+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:55:18.477971+0000","last_undegraded":"2024-09-13T14:55:18.477971+0000","last_fullsized":"2024-09-13T14:55:18.477971+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482472+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:55:18.482472+0000","last_peered":"2024-09-13T14:55:18.482472+0000","last_clean":"2024-09-13T14:55:18.482472+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:55:18.482472+0000","last_undegraded":"2024-09-13T14:55:18.482472+0000","last_fullsized":"2024-09-13T14:55:18.482472+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":768,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.520450+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:55:19.520450+0000","last_peered":"2024-09-13T14:55:19.520450+0000","last_clean":"2024-09-13T14:55:19.520450+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:55:19.520450+0000","last_undegraded":"2024-09-13T14:55:19.520450+0000","last_fullsized":"2024-09-13T14:55:19.520450+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"459'5395","reported_seq":5413,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.223954+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:55:33.223954+0000","last_peered":"2024-09-13T14:55:33.223954+0000","last_clean":"2024-09-13T14:55:33.223954+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:55:33.223954+0000","last_undegraded":"2024-09-13T14:55:33.223954+0000","last_fullsized":"2024-09-13T14:55:33.223954+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5395,"log_dups_size":0,"ondisk_log_size":5395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353566720,"num_objects":5395,"num_object_clones":0,"num_object_copies":16185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5395,"num_write_kb":345280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486341+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:55:18.486341+0000","last_peered":"2024-09-13T14:55:18.486341+0000","last_clean":"2024-09-13T14:55:18.486341+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:55:18.486341+0000","last_undegraded":"2024-09-13T14:55:18.486341+0000","last_fullsized":"2024-09-13T14:55:18.486341+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":404,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478257+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:55:18.478257+0000","last_peered":"2024-09-13T14:55:18.478257+0000","last_clean":"2024-09-13T14:55:18.478257+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:55:18.478257+0000","last_undegraded":"2024-09-13T14:55:18.478257+0000","last_fullsized":"2024-09-13T14:55:18.478257+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1431,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522229+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:55:19.522229+0000","last_peered":"2024-09-13T14:55:19.522229+0000","last_clean":"2024-09-13T14:55:19.522229+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:55:19.522229+0000","last_undegraded":"2024-09-13T14:55:19.522229+0000","last_fullsized":"2024-09-13T14:55:19.522229+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":749,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519744+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:55:19.519744+0000","last_peered":"2024-09-13T14:55:19.519744+0000","last_clean":"2024-09-13T14:55:19.519744+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:55:19.519744+0000","last_undegraded":"2024-09-13T14:55:19.519744+0000","last_fullsized":"2024-09-13T14:55:19.519744+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"459'5264","reported_seq":5282,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.815932+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:55:32.815932+0000","last_peered":"2024-09-13T14:55:32.815932+0000","last_clean":"2024-09-13T14:55:32.815932+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:55:32.815932+0000","last_undegraded":"2024-09-13T14:55:32.815932+0000","last_fullsized":"2024-09-13T14:55:32.815932+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344981504,"num_objects":5264,"num_object_clones":0,"num_object_copies":15792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5264,"num_write_kb":336896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":478,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.485875+0000","last_change":"2024-09-13T14:53:08.775380+0000","last_active":"2024-09-13T14:55:18.485875+0000","last_peered":"2024-09-13T14:55:18.485875+0000","last_clean":"2024-09-13T14:55:18.485875+0000","last_became_active":"2024-09-13T14:53:08.775030+0000","last_became_peered":"2024-09-13T14:53:08.775030+0000","last_unstale":"2024-09-13T14:55:18.485875+0000","last_undegraded":"2024-09-13T14:55:18.485875+0000","last_fullsized":"2024-09-13T14:55:18.485875+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:57.680954+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":758,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522139+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:55:19.522139+0000","last_peered":"2024-09-13T14:55:19.522139+0000","last_clean":"2024-09-13T14:55:19.522139+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:55:19.522139+0000","last_undegraded":"2024-09-13T14:55:19.522139+0000","last_fullsized":"2024-09-13T14:55:19.522139+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":875,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481404+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:55:18.481404+0000","last_peered":"2024-09-13T14:55:18.481404+0000","last_clean":"2024-09-13T14:55:18.481404+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:55:18.481404+0000","last_undegraded":"2024-09-13T14:55:18.481404+0000","last_fullsized":"2024-09-13T14:55:18.481404+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515153+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:55:19.515153+0000","last_peered":"2024-09-13T14:55:19.515153+0000","last_clean":"2024-09-13T14:55:19.515153+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:55:19.515153+0000","last_undegraded":"2024-09-13T14:55:19.515153+0000","last_fullsized":"2024-09-13T14:55:19.515153+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"459'5379","reported_seq":5397,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.916042+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:55:32.916042+0000","last_peered":"2024-09-13T14:55:32.916042+0000","last_clean":"2024-09-13T14:55:32.916042+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:55:32.916042+0000","last_undegraded":"2024-09-13T14:55:32.916042+0000","last_fullsized":"2024-09-13T14:55:32.916042+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352518144,"num_objects":5379,"num_object_clones":0,"num_object_copies":16137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5379,"num_write_kb":344256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481777+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:55:18.481777+0000","last_peered":"2024-09-13T14:55:18.481777+0000","last_clean":"2024-09-13T14:55:18.481777+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:55:18.481777+0000","last_undegraded":"2024-09-13T14:55:18.481777+0000","last_fullsized":"2024-09-13T14:55:18.481777+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":816,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.518437+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:55:19.518437+0000","last_peered":"2024-09-13T14:55:19.518437+0000","last_clean":"2024-09-13T14:55:19.518437+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:55:19.518437+0000","last_undegraded":"2024-09-13T14:55:19.518437+0000","last_fullsized":"2024-09-13T14:55:19.518437+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1134,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521764+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:55:19.521764+0000","last_peered":"2024-09-13T14:55:19.521764+0000","last_clean":"2024-09-13T14:55:19.521764+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:55:19.521764+0000","last_undegraded":"2024-09-13T14:55:19.521764+0000","last_fullsized":"2024-09-13T14:55:19.521764+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":806,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.512838+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:55:19.512838+0000","last_peered":"2024-09-13T14:55:19.512838+0000","last_clean":"2024-09-13T14:55:19.512838+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:55:19.512838+0000","last_undegraded":"2024-09-13T14:55:19.512838+0000","last_fullsized":"2024-09-13T14:55:19.512838+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"459'5357","reported_seq":5375,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:32.818193+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:55:32.818193+0000","last_peered":"2024-09-13T14:55:32.818193+0000","last_clean":"2024-09-13T14:55:32.818193+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:55:32.818193+0000","last_undegraded":"2024-09-13T14:55:32.818193+0000","last_fullsized":"2024-09-13T14:55:32.818193+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351076352,"num_objects":5357,"num_object_clones":0,"num_object_copies":16071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5357,"num_write_kb":342848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":647,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.512813+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:55:19.512813+0000","last_peered":"2024-09-13T14:55:19.512813+0000","last_clean":"2024-09-13T14:55:19.512813+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:55:19.512813+0000","last_undegraded":"2024-09-13T14:55:19.512813+0000","last_fullsized":"2024-09-13T14:55:19.512813+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:04:19.836641+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":97,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515742+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:55:19.515742+0000","last_peered":"2024-09-13T14:55:19.515742+0000","last_clean":"2024-09-13T14:55:19.515742+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:55:19.515742+0000","last_undegraded":"2024-09-13T14:55:19.515742+0000","last_fullsized":"2024-09-13T14:55:19.515742+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":877,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486505+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:55:18.486505+0000","last_peered":"2024-09-13T14:55:18.486505+0000","last_clean":"2024-09-13T14:55:18.486505+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:55:18.486505+0000","last_undegraded":"2024-09-13T14:55:18.486505+0000","last_fullsized":"2024-09-13T14:55:18.486505+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480489+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:55:18.480489+0000","last_peered":"2024-09-13T14:55:18.480489+0000","last_clean":"2024-09-13T14:55:18.480489+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:55:18.480489+0000","last_undegraded":"2024-09-13T14:55:18.480489+0000","last_fullsized":"2024-09-13T14:55:18.480489+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482533+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:55:18.482533+0000","last_peered":"2024-09-13T14:55:18.482533+0000","last_clean":"2024-09-13T14:55:18.482533+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:55:18.482533+0000","last_undegraded":"2024-09-13T14:55:18.482533+0000","last_fullsized":"2024-09-13T14:55:18.482533+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":167,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482096+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:55:18.482096+0000","last_peered":"2024-09-13T14:55:18.482096+0000","last_clean":"2024-09-13T14:55:18.482096+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:55:18.482096+0000","last_undegraded":"2024-09-13T14:55:18.482096+0000","last_fullsized":"2024-09-13T14:55:18.482096+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":745,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519621+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:55:19.519621+0000","last_peered":"2024-09-13T14:55:19.519621+0000","last_clean":"2024-09-13T14:55:19.519621+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:55:19.519621+0000","last_undegraded":"2024-09-13T14:55:19.519621+0000","last_fullsized":"2024-09-13T14:55:19.519621+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.481200+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:55:18.481200+0000","last_peered":"2024-09-13T14:55:18.481200+0000","last_clean":"2024-09-13T14:55:18.481200+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:55:18.481200+0000","last_undegraded":"2024-09-13T14:55:18.481200+0000","last_fullsized":"2024-09-13T14:55:18.481200+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.477993+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:55:18.477993+0000","last_peered":"2024-09-13T14:55:18.477993+0000","last_clean":"2024-09-13T14:55:18.477993+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:55:18.477993+0000","last_undegraded":"2024-09-13T14:55:18.477993+0000","last_fullsized":"2024-09-13T14:55:18.477993+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":750,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513192+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:55:19.513192+0000","last_peered":"2024-09-13T14:55:19.513192+0000","last_clean":"2024-09-13T14:55:19.513192+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:55:19.513192+0000","last_undegraded":"2024-09-13T14:55:19.513192+0000","last_fullsized":"2024-09-13T14:55:19.513192+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515906+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:55:19.515906+0000","last_peered":"2024-09-13T14:55:19.515906+0000","last_clean":"2024-09-13T14:55:19.515906+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:55:19.515906+0000","last_undegraded":"2024-09-13T14:55:19.515906+0000","last_fullsized":"2024-09-13T14:55:19.515906+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":806,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.512743+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:55:19.512743+0000","last_peered":"2024-09-13T14:55:19.512743+0000","last_clean":"2024-09-13T14:55:19.512743+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:55:19.512743+0000","last_undegraded":"2024-09-13T14:55:19.512743+0000","last_fullsized":"2024-09-13T14:55:19.512743+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":456,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.486410+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:55:18.486410+0000","last_peered":"2024-09-13T14:55:18.486410+0000","last_clean":"2024-09-13T14:55:18.486410+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:55:18.486410+0000","last_undegraded":"2024-09-13T14:55:18.486410+0000","last_fullsized":"2024-09-13T14:55:18.486410+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":838,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.512978+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:55:19.512978+0000","last_peered":"2024-09-13T14:55:19.512978+0000","last_clean":"2024-09-13T14:55:19.512978+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:55:19.512978+0000","last_undegraded":"2024-09-13T14:55:19.512978+0000","last_fullsized":"2024-09-13T14:55:19.512978+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515811+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:55:19.515811+0000","last_peered":"2024-09-13T14:55:19.515811+0000","last_clean":"2024-09-13T14:55:19.515811+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:55:19.515811+0000","last_undegraded":"2024-09-13T14:55:19.515811+0000","last_fullsized":"2024-09-13T14:55:19.515811+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":770,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.519374+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:55:19.519374+0000","last_peered":"2024-09-13T14:55:19.519374+0000","last_clean":"2024-09-13T14:55:19.519374+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:55:19.519374+0000","last_undegraded":"2024-09-13T14:55:19.519374+0000","last_fullsized":"2024-09-13T14:55:19.519374+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":819,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.518183+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:55:19.518183+0000","last_peered":"2024-09-13T14:55:19.518183+0000","last_clean":"2024-09-13T14:55:19.518183+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:55:19.518183+0000","last_undegraded":"2024-09-13T14:55:19.518183+0000","last_fullsized":"2024-09-13T14:55:19.518183+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":156,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480478+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:55:18.480478+0000","last_peered":"2024-09-13T14:55:18.480478+0000","last_clean":"2024-09-13T14:55:18.480478+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:55:18.480478+0000","last_undegraded":"2024-09-13T14:55:18.480478+0000","last_fullsized":"2024-09-13T14:55:18.480478+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":716,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522362+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:55:19.522362+0000","last_peered":"2024-09-13T14:55:19.522362+0000","last_clean":"2024-09-13T14:55:19.522362+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:55:19.522362+0000","last_undegraded":"2024-09-13T14:55:19.522362+0000","last_fullsized":"2024-09-13T14:55:19.522362+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":92,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515244+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:55:19.515244+0000","last_peered":"2024-09-13T14:55:19.515244+0000","last_clean":"2024-09-13T14:55:19.515244+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:55:19.515244+0000","last_undegraded":"2024-09-13T14:55:19.515244+0000","last_fullsized":"2024-09-13T14:55:19.515244+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482408+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:55:18.482408+0000","last_peered":"2024-09-13T14:55:18.482408+0000","last_clean":"2024-09-13T14:55:18.482408+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:55:18.482408+0000","last_undegraded":"2024-09-13T14:55:18.482408+0000","last_fullsized":"2024-09-13T14:55:18.482408+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":424,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478071+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:55:18.478071+0000","last_peered":"2024-09-13T14:55:18.478071+0000","last_clean":"2024-09-13T14:55:18.478071+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:55:18.478071+0000","last_undegraded":"2024-09-13T14:55:18.478071+0000","last_fullsized":"2024-09-13T14:55:18.478071+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.480554+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:55:18.480554+0000","last_peered":"2024-09-13T14:55:18.480554+0000","last_clean":"2024-09-13T14:55:18.480554+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:55:18.480554+0000","last_undegraded":"2024-09-13T14:55:18.480554+0000","last_fullsized":"2024-09-13T14:55:18.480554+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":390,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.478429+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:55:18.478429+0000","last_peered":"2024-09-13T14:55:18.478429+0000","last_clean":"2024-09-13T14:55:18.478429+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:55:18.478429+0000","last_undegraded":"2024-09-13T14:55:18.478429+0000","last_fullsized":"2024-09-13T14:55:18.478429+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":830,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513578+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:55:19.513578+0000","last_peered":"2024-09-13T14:55:19.513578+0000","last_clean":"2024-09-13T14:55:19.513578+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:55:19.513578+0000","last_undegraded":"2024-09-13T14:55:19.513578+0000","last_fullsized":"2024-09-13T14:55:19.513578+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":803,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.514628+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:55:19.514628+0000","last_peered":"2024-09-13T14:55:19.514628+0000","last_clean":"2024-09-13T14:55:19.514628+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:55:19.514628+0000","last_undegraded":"2024-09-13T14:55:19.514628+0000","last_fullsized":"2024-09-13T14:55:19.514628+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":458,"state":"active+clean","last_fresh":"2024-09-13T14:55:18.482219+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:55:18.482219+0000","last_peered":"2024-09-13T14:55:18.482219+0000","last_clean":"2024-09-13T14:55:18.482219+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:55:18.482219+0000","last_undegraded":"2024-09-13T14:55:18.482219+0000","last_fullsized":"2024-09-13T14:55:18.482219+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":99,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.515789+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:55:19.515789+0000","last_peered":"2024-09-13T14:55:19.515789+0000","last_clean":"2024-09-13T14:55:19.515789+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:55:19.515789+0000","last_undegraded":"2024-09-13T14:55:19.515789+0000","last_fullsized":"2024-09-13T14:55:19.515789+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5591990272,"num_objects":85327,"num_object_clones":0,"num_object_copies":255981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85327,"num_write_kb":5460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16635920384,"data_stored":16635920384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85327,"ondisk_log_size":85327,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398033,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2274744,"kb_used_data":1374576,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":91470408,"statfs":{"total":95995035648,"available":93665697792,"internally_reserved":0,"allocated":1407565824,"data_stored":1405121334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594261,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3266600,"kb_used_data":2372128,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":90478552,"statfs":{"total":95995035648,"available":92650037248,"internally_reserved":0,"allocated":2429059072,"data_stored":2426630264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665715,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2626852,"kb_used_data":1709148,"kb_used_omap":9,"kb_used_meta":917686,"kb_avail":91118300,"statfs":{"total":95995035648,"available":93305139200,"internally_reserved":0,"allocated":1750167552,"data_stored":1747739147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":939710664},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986717,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3614836,"kb_used_data":2723308,"kb_used_omap":1,"kb_used_meta":891518,"kb_avail":90130316,"statfs":{"total":95995035648,"available":92293443584,"internally_reserved":0,"allocated":2788667392,"data_stored":2786226928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":912914776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697276,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2710692,"kb_used_data":1671196,"kb_used_omap":4,"kb_used_meta":1039483,"kb_avail":91034460,"statfs":{"total":95995035648,"available":93219287040,"internally_reserved":0,"allocated":1711304704,"data_stored":1708876380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1064430667},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860803,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2807720,"kb_used_data":2006112,"kb_used_omap":8,"kb_used_meta":801591,"kb_avail":90937432,"statfs":{"total":95995035648,"available":93119930368,"internally_reserved":0,"allocated":2054258688,"data_stored":2051826782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":820830028},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020474,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1619936,"kb_used_data":1009240,"kb_used_omap":9,"kb_used_meta":610678,"kb_avail":92125216,"statfs":{"total":95995035648,"available":94336221184,"internally_reserved":0,"allocated":1033461760,"data_stored":1031036909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":625335109},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187865,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4264544,"kb_used_data":3379352,"kb_used_omap":7,"kb_used_meta":885176,"kb_avail":89480608,"statfs":{"total":95995035648,"available":91628142592,"internally_reserved":0,"allocated":3460456448,"data_stored":3458031863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":906420555},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3464757248,"data_stored":3464757248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1024917504,"data_stored":1024917504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2048786432,"data_stored":2048786432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1675231232,"data_stored":1675231232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2792423424,"data_stored":2792423424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1768685568,"data_stored":1768685568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2455306240,"data_stored":2455306240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1405812736,"data_stored":1405812736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:36.756 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:55:36.757 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:55:36.757 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:55:36.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:36.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:36 smithi189 ceph-mon[90992]: from='client.47021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:36.758 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:36 smithi189 ceph-mon[90992]: pgmap v1503: 99 pgs: 99 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:55:36.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:36.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:36.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 90290 90274 104.471 72.875 0.00831717 0.00956631 2024-09-13T14:55:37.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:37.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:37.154 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:37.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:37.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:37.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:37.532 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:37.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:37.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:37.673 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:37] "GET /metrics HTTP/1.1" 200 36719 "" "Prometheus/2.51.0" 2024-09-13T14:55:37.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.749 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:37.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:37.850 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:37.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:55:37.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:38.000 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 91147 91131 103.545 53.5625 0.00955591 0.00965272 2024-09-13T14:55:38.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:38.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:38.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:38.287 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:38.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:38.425 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.457 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.526 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155671 9.13 9:cb1bd516:::benchmark_data_smithi017_339564_object36198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.68503997100001, 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.922842885, 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.527 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155671 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:55:38.528 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836826+0000", 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836834+0000", 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9310000000000008e-06 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.529 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836852+0000", 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8045000000000001e-05 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836858+0000", 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1260000000000003e-06 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.530 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836923+0000", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5086999999999995e-05 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836934+0000", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0295999999999999e-05 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.749616+0000", 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91268234299999995 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.531 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.749707+0000", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0661000000000005e-05 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.750137+0000", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00043054600000000002 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753267+0000", 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031302109999999999 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.532 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759531+0000", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062633430000000002 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759578+0000", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7525999999999999e-05 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759626+0000", 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7190000000000001e-05 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.533 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759671+0000", 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5284999999999998e-05 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155725 9.13 9:cf82d474:::benchmark_data_smithi017_339564_object36252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.636086697, 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87416029900000003, 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.534 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155725 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.535 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885780+0000", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885786+0000", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0739999999999998e-06 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.536 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885796+0000", 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0159999999999999e-05 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885803+0000", 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7410000000000003e-06 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885880+0000", 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7766e-05 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.537 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885891+0000", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0491e-05 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753309+0000", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86741857200000005 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753429+0000", 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000119943 2024-09-13T14:55:38.538 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753775+0000", 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034539599999999998 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759713+0000", 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059384269999999996 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.539 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759810+0000", 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7238000000000007e-05 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759854+0000", 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3268e-05 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759900+0000", 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6601999999999999e-05 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.540 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759942+0000", 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1205999999999998e-05 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155861 9.13 9:cb69ae69:::benchmark_data_smithi017_339564_object36388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.51061837, 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75462436099999997, 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.541 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155861 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.542 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011248+0000", 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011255+0000", 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6350000000000006e-06 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.543 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011269+0000", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3552e-05 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011275+0000", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5579999999999997e-06 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011328+0000", 2024-09-13T14:55:38.544 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2098999999999997e-05 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011339+0000", 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0995000000000001e-05 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756394+0000", 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74505518699999995 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.545 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756484+0000", 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0662999999999996e-05 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756876+0000", 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00039175800000000003 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759991+0000", 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031153069999999999 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.546 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.762998+0000", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003006646 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765772+0000", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027739549999999998 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765832+0000", 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9579e-05 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.547 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765874+0000", 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2277999999999997e-05 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155867 9.13 9:cc843ee5:::benchmark_data_smithi017_339564_object36394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.50794873999999, 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76626357199999995, 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.548 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155867 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:55:38.549 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013918+0000", 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013923+0000", 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1409999999999998e-06 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.550 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013931+0000", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5890000000000003e-06 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013937+0000", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6420000000000004e-06 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013987+0000", 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0076e-05 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.551 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013998+0000", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1126e-05 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760009+0000", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74601124900000004 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760148+0000", 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013879100000000001 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.552 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760558+0000", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00040973499999999999 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772815+0000", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012256800999999999 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772896+0000", 2024-09-13T14:55:38.553 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1201999999999999e-05 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780071+0000", 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071757339999999996 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780126+0000", 2024-09-13T14:55:38.554 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5019999999999998e-05 2024-09-13T14:55:38.555 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.555 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.555 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.555 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780183+0000", 2024-09-13T14:55:38.555 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6257e-05 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155874 9.13 9:cacd9d19:::benchmark_data_smithi017_339564_object36401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.49978897299999, 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75826068499999999, 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.556 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155874 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.557 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022078+0000", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022082+0000", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8249999999999998e-06 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022091+0000", 2024-09-13T14:55:38.558 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7590000000000003e-06 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022095+0000", 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5719999999999996e-06 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022151+0000", 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6004000000000001e-05 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.559 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022163+0000", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1184e-05 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763022+0000", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74085988400000002 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763131+0000", 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010882699999999999 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.560 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763494+0000", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036248500000000001 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772846+0000", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0093522559999999998 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772933+0000", 2024-09-13T14:55:38.561 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7355999999999995e-05 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780239+0000", 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073059930000000002 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780269+0000", 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0113000000000001e-05 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.562 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780340+0000", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0186999999999997e-05 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155911 9.13 9:c90ad918:::benchmark_data_smithi017_339564_object36438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.46676944000001, 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.725374674, 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.563 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155911 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.564 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055097+0000", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055113+0000", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6109000000000001e-05 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055131+0000", 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7592e-05 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.565 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055136+0000", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8289999999999996e-06 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055304+0000", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000167226 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055313+0000", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8989999999999996e-06 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.566 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765911+0000", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71059752200000004 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.766019+0000", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010836900000000001 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.766446+0000", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042646300000000003 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772886+0000", 2024-09-13T14:55:38.567 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064403350000000002 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772937+0000", 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1322000000000001e-05 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780396+0000", 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074582590000000001 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.568 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780439+0000", 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3310999999999999e-05 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780473+0000", 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4119999999999999e-05 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155920 9.13 9:c9d1c594:::benchmark_data_smithi017_339564_object36447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:55:38.569 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.46011813600001, 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71884768499999996, 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155920 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.570 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061749+0000", 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.571 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061755+0000", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9710000000000001e-06 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061765+0000", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9739999999999997e-06 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061771+0000", 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0390000000000001e-06 2024-09-13T14:55:38.572 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061846+0000", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5196000000000003e-05 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061856+0000", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0091999999999999e-05 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.769458+0000", 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70760195599999998 2024-09-13T14:55:38.573 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.769570+0000", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011196000000000001 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.770032+0000", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000462285 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772928+0000", 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028957190000000002 2024-09-13T14:55:38.574 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772941+0000", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3030999999999999e-05 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780534+0000", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075933520000000003 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780566+0000", 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1912000000000002e-05 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.575 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780597+0000", 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1151e-05 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155925 9.13 9:cbc0753f:::benchmark_data_smithi017_339564_object36452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.455575455, 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.72457860200000002, 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.576 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155925 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.577 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066291+0000", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066298+0000", 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8480000000000003e-06 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.578 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066307+0000", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2660000000000007e-06 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066312+0000", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0150000000000003e-06 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066367+0000", 2024-09-13T14:55:38.579 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5371000000000002e-05 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066381+0000", 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4102e-05 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772961+0000", 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70658006699999998 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.580 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.773077+0000", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000115244 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.773527+0000", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000449901 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780508+0000", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069818889999999998 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.581 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780650+0000", 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014148399999999999 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790742+0000", 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010092131000000001 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790832+0000", 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9962000000000004e-05 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.582 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790871+0000", 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9002999999999999e-05 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155944 9.13 9:cb290f3d:::benchmark_data_smithi017_339564_object36471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.43737394199999, 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70647894899999997, 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.583 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155944 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.584 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084493+0000", 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:38.585 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084497+0000", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.455e-06 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084505+0000", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7370000000000006e-06 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084509+0000", 2024-09-13T14:55:38.586 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.228e-06 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084552+0000", 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3189999999999998e-05 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084561+0000", 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9779999999999994e-06 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776212+0000", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69165086600000003 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776296+0000", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3761e-05 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776736+0000", 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00044013400000000001 2024-09-13T14:55:38.588 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780625+0000", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038895140000000002 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780677+0000", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1394000000000003e-05 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790879+0000", 2024-09-13T14:55:38.589 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010202573 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790938+0000", 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9088000000000001e-05 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790973+0000", 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4690999999999997e-05 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.590 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155948 9.13 9:c80b3a9a:::benchmark_data_smithi017_339564_object36475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 198.433518257, 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71039708300000004, 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155948 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.591 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088349+0000", 2024-09-13T14:55:38.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088352+0000", 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3859999999999999e-06 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088364+0000", 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1943000000000001e-05 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088369+0000", 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7129999999999996e-06 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088416+0000", 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6897000000000002e-05 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088423+0000", 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8619999999999997e-06 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780695+0000", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69227191700000001 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780783+0000", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7839999999999999e-05 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.781237+0000", 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00045437599999999999 2024-09-13T14:55:38.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.791023+0000", 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0097855939999999999 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798583+0000", 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075606090000000003 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.596 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.597 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798663+0000", 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9684999999999993e-05 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798712+0000", 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8634999999999998e-05 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798747+0000", 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5209000000000001e-05 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:234956 9.15 9:a9c4d06b:::benchmark_data_smithi017_339564_object115483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.827682341, 2024-09-13T14:55:38.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94212623900000003, 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 234956 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694184+0000", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694190+0000", 2024-09-13T14:55:38.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9440000000000002e-06 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694204+0000", 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3579e-05 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694209+0000", 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4559999999999999e-06 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694263+0000", 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3343000000000001e-05 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694272+0000", 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4550000000000001e-06 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611453+0000", 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91718084799999999 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611537+0000", 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3949000000000005e-05 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611945+0000", 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00040808199999999999 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.604 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616712+0000", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047666710000000001 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636236+0000", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019524308000000001 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636262+0000", 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5935000000000002e-05 2024-09-13T14:55:38.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636290+0000", 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8626999999999999e-05 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636312+0000", 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1372000000000001e-05 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.606 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235016 9.15 9:a9ffcdb4:::benchmark_data_smithi017_339564_object115543:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.775567205, 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.890119416, 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235016 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746300+0000", 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:38.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746306+0000", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8229999999999997e-06 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746318+0000", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2009e-05 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746325+0000", 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4420000000000003e-06 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746861+0000", 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053583600000000004 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746869+0000", 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2269999999999993e-06 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.613740+0000", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86687114600000004 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.613811+0000", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.148e-05 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.614076+0000", 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00026431000000000002 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618157+0000", 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004081459 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636349+0000", 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018191807000000001 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.612 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636383+0000", 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3850000000000003e-05 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636404+0000", 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1271999999999999e-05 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636420+0000", 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5965e-05 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.613 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235041 9.15 9:ae766b01:::benchmark_data_smithi017_339564_object115568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.752477074, 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86726312000000005, 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235041 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.614 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769390+0000", 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.615 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769396+0000", 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7219999999999996e-06 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769406+0000", 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0538e-05 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.616 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769412+0000", 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7130000000000002e-06 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771644+0000", 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002232506 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771739+0000", 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4989999999999997e-05 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.617 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615272+0000", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84353275999999999 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615337+0000", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4394999999999996e-05 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615557+0000", 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022054600000000001 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.618 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622593+0000", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070358640000000002 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636445+0000", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013852491999999999 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636557+0000", 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111919 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.619 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636621+0000", 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4115999999999994e-05 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636654+0000", 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2498e-05 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.620 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235042 9.15 9:a8280366:::benchmark_data_smithi017_339564_object115569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.75183072199999, 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.866749561, 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235042 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.621 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770036+0000", 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.622 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770041+0000", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1560000000000003e-06 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770057+0000", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5051999999999999e-05 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770063+0000", 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2519999999999996e-06 2024-09-13T14:55:38.623 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771761+0000", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001698101 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771767+0000", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7259999999999999e-06 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616749+0000", 2024-09-13T14:55:38.624 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84498218899999999 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616807+0000", 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8288999999999997e-05 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.617048+0000", 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024090799999999999 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.625 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625508+0000", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084598209999999993 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636701+0000", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011193053999999999 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636734+0000", 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3061000000000001e-05 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.626 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636757+0000", 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3300000000000001e-05 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636787+0000", 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9543000000000002e-05 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.627 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235059 9.15 9:ab89eb0d:::benchmark_data_smithi017_339564_object115586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.73541280000001, 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.85055655600000002, 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235059 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.628 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786454+0000", 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.629 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786459+0000", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1329999999999998e-06 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786469+0000", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2299999999999997e-06 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786474+0000", 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8620000000000002e-06 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.630 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786521+0000", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7814e-05 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786530+0000", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8040000000000007e-06 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618177+0000", 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83164686700000001 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.631 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618251+0000", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4198999999999999e-05 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618489+0000", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000237875 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625510+0000", 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070213799999999998 2024-09-13T14:55:38.632 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636836+0000", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011325426 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636936+0000", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000100246 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636979+0000", 2024-09-13T14:55:38.633 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3198999999999998e-05 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637012+0000", 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2322e-05 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235072 9.15 9:ad20fbe1:::benchmark_data_smithi017_339564_object115599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:55:38.634 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.723809108, 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83910899999999999, 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235072 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.635 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798058+0000", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798064+0000", 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:55:38.636 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798075+0000", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1813e-05 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798081+0000", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3900000000000001e-06 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798135+0000", 2024-09-13T14:55:38.637 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4280999999999997e-05 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798144+0000", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6780000000000006e-06 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619664+0000", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.82152004199999995 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619724+0000", 2024-09-13T14:55:38.638 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9849999999999998e-05 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619938+0000", 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021460899999999999 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625513+0000", 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055745090000000001 2024-09-13T14:55:38.639 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.640 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.640 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.640 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636890+0000", 2024-09-13T14:55:38.640 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011376828 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637105+0000", 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021505100000000001 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637133+0000", 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8121999999999999e-05 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.641 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637168+0000", 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4981000000000003e-05 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235104 9.15 9:ad7f5d5c:::benchmark_data_smithi017_339564_object115631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.69144339100001, 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80683303200000001, 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.642 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235104 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.643 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830424+0000", 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830428+0000", 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8120000000000001e-06 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.644 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830436+0000", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0900000000000005e-06 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830440+0000", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6729999999999998e-06 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830481+0000", 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1090000000000001e-05 2024-09-13T14:55:38.645 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830487+0000", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6869999999999997e-06 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621047+0000", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.790559503 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621112+0000", 2024-09-13T14:55:38.646 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5216999999999996e-05 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621423+0000", 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00031051599999999999 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.628538+0000", 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071157879999999996 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.647 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637070+0000", 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085320399999999994 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637199+0000", 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012852299999999999 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637224+0000", 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5612000000000001e-05 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.648 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637258+0000", 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3073e-05 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235132 9.15 9:ae3e20fb:::benchmark_data_smithi017_339564_object115659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.660797203, 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.77630533999999995, 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.649 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235132 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:55:38.650 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861069+0000", 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861076+0000", 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1179999999999997e-06 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.651 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861086+0000", 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.04e-05 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861091+0000", 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.862798+0000", 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001706611 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.652 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.862806+0000", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4090000000000003e-06 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622623+0000", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75981690300000004 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622690+0000", 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6616000000000001e-05 2024-09-13T14:55:38.653 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622966+0000", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00027669599999999999 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.628541+0000", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005574603 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637286+0000", 2024-09-13T14:55:38.654 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0087450630000000008 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637309+0000", 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3176e-05 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637332+0000", 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3040999999999998e-05 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.655 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637376+0000", 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3698000000000001e-05 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235144 9.15 9:af6e4b6c:::benchmark_data_smithi017_339564_object115671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.64637873700001, 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76209903800000001, 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.656 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235144 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.657 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875488+0000", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875494+0000", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2700000000000004e-06 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875504+0000", 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0808999999999999e-05 2024-09-13T14:55:38.658 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875509+0000", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3660000000000002e-06 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875549+0000", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0790000000000001e-05 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875555+0000", 2024-09-13T14:55:38.659 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8629999999999996e-06 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624121+0000", 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.748565641 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624186+0000", 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5436000000000002e-05 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.660 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624422+0000", 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000235236 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.631482+0000", 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070606430000000001 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637431+0000", 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059484849999999999 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.661 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637507+0000", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6427999999999994e-05 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637549+0000", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1567999999999999e-05 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637588+0000", 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9459000000000003e-05 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.662 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235167 9.15 9:a8ab814f:::benchmark_data_smithi017_339564_object115694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 104.61941828499999, 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.73526514799999998, 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235167 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.663 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902448+0000", 2024-09-13T14:55:38.664 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902455+0000", 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4629999999999998e-06 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902471+0000", 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5750999999999999e-05 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:55:38.665 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902477+0000", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.857e-06 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902526+0000", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8964000000000001e-05 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902534+0000", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.456e-06 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:55:38.666 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625559+0000", 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.72302467800000003 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625624+0000", 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5321999999999996e-05 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625900+0000", 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00027566800000000002 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.667 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.631502+0000", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0056024730000000002 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637469+0000", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059668270000000001 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637640+0000", 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017078699999999999 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.668 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637681+0000", 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1028000000000001e-05 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637715+0000", 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4029999999999998e-05 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:55:38.669 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:55:38.685 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:38.755 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:38 smithi189 ceph-mon[90992]: pgmap v1504: 99 pgs: 99 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:55:38.787 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:38.800 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:38 smithi017 ceph-mon[107556]: pgmap v1504: 99 pgs: 99 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:55:38.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:38 smithi017 ceph-mon[103001]: pgmap v1504: 99 pgs: 99 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:55:38.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:38.905 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:38.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:38.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:38.998 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 92255 92239 102.931 69.25 0.0184902 0.00971052 2024-09-13T14:55:39.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:39.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:39.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:39.211 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.249 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:39.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:39.389 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:39.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:55:39.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:39.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:39.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:39.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:39.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:39.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.937 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:55:39.943 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:55:39.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:39.988 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:40.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 93273 93257 102.235 63.625 0.0168043 0.00977624 2024-09-13T14:55:40.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:40.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:40.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:40.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:40.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:40.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:40.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:40.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:40.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:40.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:40.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:40.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:40 smithi017 ceph-mon[103001]: pgmap v1505: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:40.769 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:40 smithi017 ceph-mon[107556]: pgmap v1505: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:40.800 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:40.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:40.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:40.978 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:41.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 94471 94455 101.763 74.875 0.00652489 0.00982191 2024-09-13T14:55:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:40 smithi189 ceph-mon[90992]: pgmap v1505: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:41.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:41.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:41.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:41.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:41.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:41.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:41.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:41.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:41.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.774 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1505,"stamp":"2024-09-13T14:55:39.426409+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5964109773,"num_objects":90960,"num_object_clones":0,"num_object_copies":272880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90960,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":91839,"num_write_kb":5832271,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":8394258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93064,"ondisk_log_size":93064,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24413160,"kb_used_data":17411236,"kb_used_omap":44,"kb_used_meta":7001811,"kb_avail":725548056,"statfs":{"total":767960285184,"available":742961209344,"internally_reserved":0,"allocated":17829105664,"data_stored":17809622199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7169854711},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2,1,6,1,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":682491904,"num_objects":10414,"num_object_clones":0,"num_object_copies":31242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10414,"num_write_kb":666496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005185"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":808,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.511031+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:55:33.511031+0000","last_peered":"2024-09-13T14:55:33.511031+0000","last_clean":"2024-09-13T14:55:33.511031+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:55:33.511031+0000","last_undegraded":"2024-09-13T14:55:33.511031+0000","last_fullsized":"2024-09-13T14:55:33.511031+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":458,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521891+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:55:19.521891+0000","last_peered":"2024-09-13T14:55:19.521891+0000","last_clean":"2024-09-13T14:55:19.521891+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:55:19.521891+0000","last_undegraded":"2024-09-13T14:55:19.521891+0000","last_fullsized":"2024-09-13T14:55:19.521891+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":387,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509772+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:55:19.509772+0000","last_peered":"2024-09-13T14:55:19.509772+0000","last_clean":"2024-09-13T14:55:19.509772+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:55:19.509772+0000","last_undegraded":"2024-09-13T14:55:19.509772+0000","last_fullsized":"2024-09-13T14:55:19.509772+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.503134+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:55:33.503134+0000","last_peered":"2024-09-13T14:55:33.503134+0000","last_clean":"2024-09-13T14:55:33.503134+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:55:33.503134+0000","last_undegraded":"2024-09-13T14:55:33.503134+0000","last_fullsized":"2024-09-13T14:55:33.503134+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.504574+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:55:33.504574+0000","last_peered":"2024-09-13T14:55:33.504574+0000","last_clean":"2024-09-13T14:55:33.504574+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:55:33.504574+0000","last_undegraded":"2024-09-13T14:55:33.504574+0000","last_fullsized":"2024-09-13T14:55:33.504574+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":427,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.525308+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:55:19.525308+0000","last_peered":"2024-09-13T14:55:19.525308+0000","last_clean":"2024-09-13T14:55:19.525308+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:55:19.525308+0000","last_undegraded":"2024-09-13T14:55:19.525308+0000","last_fullsized":"2024-09-13T14:55:19.525308+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":808,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.513481+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:55:33.513481+0000","last_peered":"2024-09-13T14:55:33.513481+0000","last_clean":"2024-09-13T14:55:33.513481+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:55:33.513481+0000","last_undegraded":"2024-09-13T14:55:33.513481+0000","last_fullsized":"2024-09-13T14:55:33.513481+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":489,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513364+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:55:19.513364+0000","last_peered":"2024-09-13T14:55:19.513364+0000","last_clean":"2024-09-13T14:55:19.513364+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:55:19.513364+0000","last_undegraded":"2024-09-13T14:55:19.513364+0000","last_fullsized":"2024-09-13T14:55:19.513364+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"460'5644","reported_seq":5663,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:38.340185+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:55:38.340185+0000","last_peered":"2024-09-13T14:55:38.340185+0000","last_clean":"2024-09-13T14:55:38.340185+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:55:38.340185+0000","last_undegraded":"2024-09-13T14:55:38.340185+0000","last_fullsized":"2024-09-13T14:55:38.340185+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369885184,"num_objects":5644,"num_object_clones":0,"num_object_copies":16932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5644,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":481,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522865+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:55:19.522865+0000","last_peered":"2024-09-13T14:55:19.522865+0000","last_clean":"2024-09-13T14:55:19.522865+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:55:19.522865+0000","last_undegraded":"2024-09-13T14:55:19.522865+0000","last_fullsized":"2024-09-13T14:55:19.522865+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":372,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.508955+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:55:19.508955+0000","last_peered":"2024-09-13T14:55:19.508955+0000","last_clean":"2024-09-13T14:55:19.508955+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:55:19.508955+0000","last_undegraded":"2024-09-13T14:55:19.508955+0000","last_fullsized":"2024-09-13T14:55:19.508955+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"460'5831","reported_seq":5849,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.695398+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:55:37.695398+0000","last_peered":"2024-09-13T14:55:37.695398+0000","last_clean":"2024-09-13T14:55:37.695398+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:55:37.695398+0000","last_undegraded":"2024-09-13T14:55:37.695398+0000","last_fullsized":"2024-09-13T14:55:37.695398+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5831,"log_dups_size":0,"ondisk_log_size":5831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382140416,"num_objects":5831,"num_object_clones":0,"num_object_copies":17493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5831,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":471,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522614+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:55:19.522614+0000","last_peered":"2024-09-13T14:55:19.522614+0000","last_clean":"2024-09-13T14:55:19.522614+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:55:19.522614+0000","last_undegraded":"2024-09-13T14:55:19.522614+0000","last_fullsized":"2024-09-13T14:55:19.522614+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517378+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:55:19.517378+0000","last_peered":"2024-09-13T14:55:19.517378+0000","last_clean":"2024-09-13T14:55:19.517378+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:55:19.517378+0000","last_undegraded":"2024-09-13T14:55:19.517378+0000","last_fullsized":"2024-09-13T14:55:19.517378+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"460'5693","reported_seq":5726,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.185308+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:55:37.185308+0000","last_peered":"2024-09-13T14:55:37.185308+0000","last_clean":"2024-09-13T14:55:37.185308+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:55:37.185308+0000","last_undegraded":"2024-09-13T14:55:37.185308+0000","last_fullsized":"2024-09-13T14:55:37.185308+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373096448,"num_objects":5693,"num_object_clones":0,"num_object_copies":17079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":425,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522304+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:55:19.522304+0000","last_peered":"2024-09-13T14:55:19.522304+0000","last_clean":"2024-09-13T14:55:19.522304+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:55:19.522304+0000","last_undegraded":"2024-09-13T14:55:19.522304+0000","last_fullsized":"2024-09-13T14:55:19.522304+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":440,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513664+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:55:19.513664+0000","last_peered":"2024-09-13T14:55:19.513664+0000","last_clean":"2024-09-13T14:55:19.513664+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:55:19.513664+0000","last_undegraded":"2024-09-13T14:55:19.513664+0000","last_fullsized":"2024-09-13T14:55:19.513664+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"460'5573","reported_seq":5606,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.222356+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:55:37.222356+0000","last_peered":"2024-09-13T14:55:37.222356+0000","last_clean":"2024-09-13T14:55:37.222356+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:55:37.222356+0000","last_undegraded":"2024-09-13T14:55:37.222356+0000","last_fullsized":"2024-09-13T14:55:37.222356+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5573,"log_dups_size":0,"ondisk_log_size":5573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5573,"num_write_kb":356672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":388,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509604+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:55:19.509604+0000","last_peered":"2024-09-13T14:55:19.509604+0000","last_clean":"2024-09-13T14:55:19.509604+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:55:19.509604+0000","last_undegraded":"2024-09-13T14:55:19.509604+0000","last_fullsized":"2024-09-13T14:55:19.509604+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.503845+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:55:33.503845+0000","last_peered":"2024-09-13T14:55:33.503845+0000","last_clean":"2024-09-13T14:55:33.503845+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:55:33.503845+0000","last_undegraded":"2024-09-13T14:55:33.503845+0000","last_fullsized":"2024-09-13T14:55:33.503845+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"460'5639","reported_seq":5672,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:35.830797+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:55:35.830797+0000","last_peered":"2024-09-13T14:55:35.830797+0000","last_clean":"2024-09-13T14:55:35.830797+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:55:35.830797+0000","last_undegraded":"2024-09-13T14:55:35.830797+0000","last_fullsized":"2024-09-13T14:55:35.830797+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369557504,"num_objects":5639,"num_object_clones":0,"num_object_copies":16917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5639,"num_write_kb":360896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":571,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.525104+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:55:19.525104+0000","last_peered":"2024-09-13T14:55:19.525104+0000","last_clean":"2024-09-13T14:55:19.525104+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:55:19.525104+0000","last_undegraded":"2024-09-13T14:55:19.525104+0000","last_fullsized":"2024-09-13T14:55:19.525104+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":840,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510004+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:55:33.510004+0000","last_peered":"2024-09-13T14:55:33.510004+0000","last_clean":"2024-09-13T14:55:33.510004+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:55:33.510004+0000","last_undegraded":"2024-09-13T14:55:33.510004+0000","last_fullsized":"2024-09-13T14:55:33.510004+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.518920+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:55:19.518920+0000","last_peered":"2024-09-13T14:55:19.518920+0000","last_clean":"2024-09-13T14:55:19.518920+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:55:19.518920+0000","last_undegraded":"2024-09-13T14:55:19.518920+0000","last_fullsized":"2024-09-13T14:55:19.518920+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"460'5631","reported_seq":5650,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:38.334698+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:55:38.334698+0000","last_peered":"2024-09-13T14:55:38.334698+0000","last_clean":"2024-09-13T14:55:38.334698+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:55:38.334698+0000","last_undegraded":"2024-09-13T14:55:38.334698+0000","last_fullsized":"2024-09-13T14:55:38.334698+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369033216,"num_objects":5631,"num_object_clones":0,"num_object_copies":16893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5631,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":285,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509038+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:55:19.509038+0000","last_peered":"2024-09-13T14:55:19.509038+0000","last_clean":"2024-09-13T14:55:19.509038+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:55:19.509038+0000","last_undegraded":"2024-09-13T14:55:19.509038+0000","last_fullsized":"2024-09-13T14:55:19.509038+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":461,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513541+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:55:19.513541+0000","last_peered":"2024-09-13T14:55:19.513541+0000","last_clean":"2024-09-13T14:55:19.513541+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:55:19.513541+0000","last_undegraded":"2024-09-13T14:55:19.513541+0000","last_fullsized":"2024-09-13T14:55:19.513541+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517767+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:55:19.517767+0000","last_peered":"2024-09-13T14:55:19.517767+0000","last_clean":"2024-09-13T14:55:19.517767+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:55:19.517767+0000","last_undegraded":"2024-09-13T14:55:19.517767+0000","last_fullsized":"2024-09-13T14:55:19.517767+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"460'5631","reported_seq":5650,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.698259+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:55:37.698259+0000","last_peered":"2024-09-13T14:55:37.698259+0000","last_clean":"2024-09-13T14:55:37.698259+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:55:37.698259+0000","last_undegraded":"2024-09-13T14:55:37.698259+0000","last_fullsized":"2024-09-13T14:55:37.698259+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369033216,"num_objects":5631,"num_object_clones":0,"num_object_copies":16893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5631,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517234+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:55:19.517234+0000","last_peered":"2024-09-13T14:55:19.517234+0000","last_clean":"2024-09-13T14:55:19.517234+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:55:19.517234+0000","last_undegraded":"2024-09-13T14:55:19.517234+0000","last_fullsized":"2024-09-13T14:55:19.517234+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517894+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:55:19.517894+0000","last_peered":"2024-09-13T14:55:19.517894+0000","last_clean":"2024-09-13T14:55:19.517894+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:55:19.517894+0000","last_undegraded":"2024-09-13T14:55:19.517894+0000","last_fullsized":"2024-09-13T14:55:19.517894+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"460'5576","reported_seq":5595,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.821671+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:55:37.821671+0000","last_peered":"2024-09-13T14:55:37.821671+0000","last_clean":"2024-09-13T14:55:37.821671+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:55:37.821671+0000","last_undegraded":"2024-09-13T14:55:37.821671+0000","last_fullsized":"2024-09-13T14:55:37.821671+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365428736,"num_objects":5576,"num_object_clones":0,"num_object_copies":16728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5576,"num_write_kb":356864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":278,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509541+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:55:19.509541+0000","last_peered":"2024-09-13T14:55:19.509541+0000","last_clean":"2024-09-13T14:55:19.509541+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:55:19.509541+0000","last_undegraded":"2024-09-13T14:55:19.509541+0000","last_fullsized":"2024-09-13T14:55:19.509541+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":810,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510022+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:55:33.510022+0000","last_peered":"2024-09-13T14:55:33.510022+0000","last_clean":"2024-09-13T14:55:33.510022+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:55:33.510022+0000","last_undegraded":"2024-09-13T14:55:33.510022+0000","last_fullsized":"2024-09-13T14:55:33.510022+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":772,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510684+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:55:33.510684+0000","last_peered":"2024-09-13T14:55:33.510684+0000","last_clean":"2024-09-13T14:55:33.510684+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:55:33.510684+0000","last_undegraded":"2024-09-13T14:55:33.510684+0000","last_fullsized":"2024-09-13T14:55:33.510684+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"460'5650","reported_seq":5669,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.702455+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:55:37.702455+0000","last_peered":"2024-09-13T14:55:37.702455+0000","last_clean":"2024-09-13T14:55:37.702455+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:55:37.702455+0000","last_undegraded":"2024-09-13T14:55:37.702455+0000","last_fullsized":"2024-09-13T14:55:37.702455+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5650,"log_dups_size":0,"ondisk_log_size":5650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370278400,"num_objects":5650,"num_object_clones":0,"num_object_copies":16950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5650,"num_write_kb":361600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":582,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521650+0000","last_change":"2024-09-13T14:53:08.772753+0000","last_active":"2024-09-13T14:55:19.521650+0000","last_peered":"2024-09-13T14:55:19.521650+0000","last_clean":"2024-09-13T14:55:19.521650+0000","last_became_active":"2024-09-13T14:53:08.772283+0000","last_became_peered":"2024-09-13T14:53:08.772283+0000","last_unstale":"2024-09-13T14:55:19.521650+0000","last_undegraded":"2024-09-13T14:55:19.521650+0000","last_fullsized":"2024-09-13T14:55:19.521650+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:06.421325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:06.421325+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:30.366156+0000","scrub_duration":0.001369898,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":984,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509925+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:55:19.509925+0000","last_peered":"2024-09-13T14:55:19.509925+0000","last_clean":"2024-09-13T14:55:19.509925+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:55:19.509925+0000","last_undegraded":"2024-09-13T14:55:19.509925+0000","last_fullsized":"2024-09-13T14:55:19.509925+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":764,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509596+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:55:33.509596+0000","last_peered":"2024-09-13T14:55:33.509596+0000","last_clean":"2024-09-13T14:55:33.509596+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:55:33.509596+0000","last_undegraded":"2024-09-13T14:55:33.509596+0000","last_fullsized":"2024-09-13T14:55:33.509596+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":426,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513694+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:55:19.513694+0000","last_peered":"2024-09-13T14:55:19.513694+0000","last_clean":"2024-09-13T14:55:19.513694+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:55:19.513694+0000","last_undegraded":"2024-09-13T14:55:19.513694+0000","last_fullsized":"2024-09-13T14:55:19.513694+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"460'5718","reported_seq":5751,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.220116+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:55:37.220116+0000","last_peered":"2024-09-13T14:55:37.220116+0000","last_clean":"2024-09-13T14:55:37.220116+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:55:37.220116+0000","last_undegraded":"2024-09-13T14:55:37.220116+0000","last_fullsized":"2024-09-13T14:55:37.220116+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374734848,"num_objects":5718,"num_object_clones":0,"num_object_copies":17154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":619,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509041+0000","last_change":"2024-09-13T14:40:26.266781+0000","last_active":"2024-09-13T14:55:33.509041+0000","last_peered":"2024-09-13T14:55:33.509041+0000","last_clean":"2024-09-13T14:55:33.509041+0000","last_became_active":"2024-09-13T14:40:26.266196+0000","last_became_peered":"2024-09-13T14:40:26.266196+0000","last_unstale":"2024-09-13T14:55:33.509041+0000","last_undegraded":"2024-09-13T14:55:33.509041+0000","last_fullsized":"2024-09-13T14:55:33.509041+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:34:47.477622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:34:47.477622+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:32.116230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":872,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521571+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:55:19.521571+0000","last_peered":"2024-09-13T14:55:19.521571+0000","last_clean":"2024-09-13T14:55:19.521571+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:55:19.521571+0000","last_undegraded":"2024-09-13T14:55:19.521571+0000","last_fullsized":"2024-09-13T14:55:19.521571+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.503469+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:55:33.503469+0000","last_peered":"2024-09-13T14:55:33.503469+0000","last_clean":"2024-09-13T14:55:33.503469+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:55:33.503469+0000","last_undegraded":"2024-09-13T14:55:33.503469+0000","last_fullsized":"2024-09-13T14:55:33.503469+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.502374+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:55:33.502374+0000","last_peered":"2024-09-13T14:55:33.502374+0000","last_clean":"2024-09-13T14:55:33.502374+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:55:33.502374+0000","last_undegraded":"2024-09-13T14:55:33.502374+0000","last_fullsized":"2024-09-13T14:55:33.502374+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"460'5782","reported_seq":5801,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:38.357989+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:55:38.357989+0000","last_peered":"2024-09-13T14:55:38.357989+0000","last_clean":"2024-09-13T14:55:38.357989+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:55:38.357989+0000","last_undegraded":"2024-09-13T14:55:38.357989+0000","last_fullsized":"2024-09-13T14:55:38.357989+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378929152,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5782,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":611,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.521590+0000","last_change":"2024-09-13T14:53:08.772285+0000","last_active":"2024-09-13T14:55:19.521590+0000","last_peered":"2024-09-13T14:55:19.521590+0000","last_clean":"2024-09-13T14:55:19.521590+0000","last_became_active":"2024-09-13T14:53:08.770613+0000","last_became_peered":"2024-09-13T14:53:08.770613+0000","last_unstale":"2024-09-13T14:55:19.521590+0000","last_undegraded":"2024-09-13T14:55:19.521590+0000","last_fullsized":"2024-09-13T14:55:19.521590+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.361942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.361942+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:08.707450+0000","scrub_duration":0.0035266220000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":790,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510434+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:55:33.510434+0000","last_peered":"2024-09-13T14:55:33.510434+0000","last_clean":"2024-09-13T14:55:33.510434+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:55:33.510434+0000","last_undegraded":"2024-09-13T14:55:33.510434+0000","last_fullsized":"2024-09-13T14:55:33.510434+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":750,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.517997+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:55:33.517997+0000","last_peered":"2024-09-13T14:55:33.517997+0000","last_clean":"2024-09-13T14:55:33.517997+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:55:33.517997+0000","last_undegraded":"2024-09-13T14:55:33.517997+0000","last_fullsized":"2024-09-13T14:55:33.517997+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":229,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.510113+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:55:19.510113+0000","last_peered":"2024-09-13T14:55:19.510113+0000","last_clean":"2024-09-13T14:55:19.510113+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:55:19.510113+0000","last_undegraded":"2024-09-13T14:55:19.510113+0000","last_fullsized":"2024-09-13T14:55:19.510113+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"460'5740","reported_seq":5759,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.666557+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:55:37.666557+0000","last_peered":"2024-09-13T14:55:37.666557+0000","last_clean":"2024-09-13T14:55:37.666557+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:55:37.666557+0000","last_undegraded":"2024-09-13T14:55:37.666557+0000","last_fullsized":"2024-09-13T14:55:37.666557+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376176640,"num_objects":5740,"num_object_clones":0,"num_object_copies":17220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5740,"num_write_kb":367360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":298,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.510443+0000","last_change":"2024-09-13T14:53:08.772053+0000","last_active":"2024-09-13T14:55:19.510443+0000","last_peered":"2024-09-13T14:55:19.510443+0000","last_clean":"2024-09-13T14:55:19.510443+0000","last_became_active":"2024-09-13T14:53:08.769631+0000","last_became_peered":"2024-09-13T14:53:08.769631+0000","last_unstale":"2024-09-13T14:55:19.510443+0000","last_undegraded":"2024-09-13T14:55:19.510443+0000","last_fullsized":"2024-09-13T14:55:19.510443+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:49:47.727247+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:53.493391+0000","scrub_duration":0.011542708,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1075,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509685+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:55:19.509685+0000","last_peered":"2024-09-13T14:55:19.509685+0000","last_clean":"2024-09-13T14:55:19.509685+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:55:19.509685+0000","last_undegraded":"2024-09-13T14:55:19.509685+0000","last_fullsized":"2024-09-13T14:55:19.509685+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517554+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:55:19.517554+0000","last_peered":"2024-09-13T14:55:19.517554+0000","last_clean":"2024-09-13T14:55:19.517554+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:55:19.517554+0000","last_undegraded":"2024-09-13T14:55:19.517554+0000","last_fullsized":"2024-09-13T14:55:19.517554+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":771,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509069+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:55:33.509069+0000","last_peered":"2024-09-13T14:55:33.509069+0000","last_clean":"2024-09-13T14:55:33.509069+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:55:33.509069+0000","last_undegraded":"2024-09-13T14:55:33.509069+0000","last_fullsized":"2024-09-13T14:55:33.509069+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"460'5726","reported_seq":5745,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:38.240679+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:55:38.240679+0000","last_peered":"2024-09-13T14:55:38.240679+0000","last_clean":"2024-09-13T14:55:38.240679+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:55:38.240679+0000","last_undegraded":"2024-09-13T14:55:38.240679+0000","last_fullsized":"2024-09-13T14:55:38.240679+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5726,"log_dups_size":0,"ondisk_log_size":5726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375259136,"num_objects":5726,"num_object_clones":0,"num_object_copies":17178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5726,"num_write_kb":366464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":459,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513709+0000","last_change":"2024-09-13T14:30:37.430211+0000","last_active":"2024-09-13T14:55:19.513709+0000","last_peered":"2024-09-13T14:55:19.513709+0000","last_clean":"2024-09-13T14:55:19.513709+0000","last_became_active":"2024-09-13T14:30:11.437455+0000","last_became_peered":"2024-09-13T14:30:11.437455+0000","last_unstale":"2024-09-13T14:55:19.513709+0000","last_undegraded":"2024-09-13T14:55:19.513709+0000","last_fullsized":"2024-09-13T14:55:19.513709+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:37.430121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:37.430121+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:08.007519+0000","scrub_duration":0.002262376,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":405,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509923+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:55:19.509923+0000","last_peered":"2024-09-13T14:55:19.509923+0000","last_clean":"2024-09-13T14:55:19.509923+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:55:19.509923+0000","last_undegraded":"2024-09-13T14:55:19.509923+0000","last_fullsized":"2024-09-13T14:55:19.509923+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1434,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509246+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:55:33.509246+0000","last_peered":"2024-09-13T14:55:33.509246+0000","last_clean":"2024-09-13T14:55:33.509246+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:55:33.509246+0000","last_undegraded":"2024-09-13T14:55:33.509246+0000","last_fullsized":"2024-09-13T14:55:33.509246+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":752,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.508975+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:55:33.508975+0000","last_peered":"2024-09-13T14:55:33.508975+0000","last_clean":"2024-09-13T14:55:33.508975+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:55:33.508975+0000","last_undegraded":"2024-09-13T14:55:33.508975+0000","last_fullsized":"2024-09-13T14:55:33.508975+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"460'5629","reported_seq":5648,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.698288+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:55:37.698288+0000","last_peered":"2024-09-13T14:55:37.698288+0000","last_clean":"2024-09-13T14:55:37.698288+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:55:37.698288+0000","last_undegraded":"2024-09-13T14:55:37.698288+0000","last_fullsized":"2024-09-13T14:55:37.698288+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902144,"num_objects":5629,"num_object_clones":0,"num_object_copies":16887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":360256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":479,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513609+0000","last_change":"2024-09-13T14:53:08.775380+0000","last_active":"2024-09-13T14:55:19.513609+0000","last_peered":"2024-09-13T14:55:19.513609+0000","last_clean":"2024-09-13T14:55:19.513609+0000","last_became_active":"2024-09-13T14:53:08.775030+0000","last_became_peered":"2024-09-13T14:53:08.775030+0000","last_unstale":"2024-09-13T14:55:19.513609+0000","last_undegraded":"2024-09-13T14:55:19.513609+0000","last_fullsized":"2024-09-13T14:55:19.513609+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:38.471038+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:38.471038+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:57.680954+0000","scrub_duration":0.0027855330000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":761,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509159+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:55:33.509159+0000","last_peered":"2024-09-13T14:55:33.509159+0000","last_clean":"2024-09-13T14:55:33.509159+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:55:33.509159+0000","last_undegraded":"2024-09-13T14:55:33.509159+0000","last_fullsized":"2024-09-13T14:55:33.509159+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":876,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522381+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:55:19.522381+0000","last_peered":"2024-09-13T14:55:19.522381+0000","last_clean":"2024-09-13T14:55:19.522381+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:55:19.522381+0000","last_undegraded":"2024-09-13T14:55:19.522381+0000","last_fullsized":"2024-09-13T14:55:19.522381+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.503100+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:55:33.503100+0000","last_peered":"2024-09-13T14:55:33.503100+0000","last_clean":"2024-09-13T14:55:33.503100+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:55:33.503100+0000","last_undegraded":"2024-09-13T14:55:33.503100+0000","last_fullsized":"2024-09-13T14:55:33.503100+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"460'5744","reported_seq":5763,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.690003+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:55:37.690003+0000","last_peered":"2024-09-13T14:55:37.690003+0000","last_clean":"2024-09-13T14:55:37.690003+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:55:37.690003+0000","last_undegraded":"2024-09-13T14:55:37.690003+0000","last_fullsized":"2024-09-13T14:55:37.690003+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5744,"log_dups_size":0,"ondisk_log_size":5744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376438784,"num_objects":5744,"num_object_clones":0,"num_object_copies":17232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5744,"num_write_kb":367616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517683+0000","last_change":"2024-09-13T14:40:26.255744+0000","last_active":"2024-09-13T14:55:19.517683+0000","last_peered":"2024-09-13T14:55:19.517683+0000","last_clean":"2024-09-13T14:55:19.517683+0000","last_became_active":"2024-09-13T14:40:26.255425+0000","last_became_peered":"2024-09-13T14:40:26.255425+0000","last_unstale":"2024-09-13T14:55:19.517683+0000","last_undegraded":"2024-09-13T14:55:19.517683+0000","last_fullsized":"2024-09-13T14:55:19.517683+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:38:14.626794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:38:14.626794+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:11.644486+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":818,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.514464+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:55:33.514464+0000","last_peered":"2024-09-13T14:55:33.514464+0000","last_clean":"2024-09-13T14:55:33.514464+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:55:33.514464+0000","last_undegraded":"2024-09-13T14:55:33.514464+0000","last_fullsized":"2024-09-13T14:55:33.514464+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1136,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510408+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:55:33.510408+0000","last_peered":"2024-09-13T14:55:33.510408+0000","last_clean":"2024-09-13T14:55:33.510408+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:55:33.510408+0000","last_undegraded":"2024-09-13T14:55:33.510408+0000","last_fullsized":"2024-09-13T14:55:33.510408+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":808,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509795+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:55:33.509795+0000","last_peered":"2024-09-13T14:55:33.509795+0000","last_clean":"2024-09-13T14:55:33.509795+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:55:33.509795+0000","last_undegraded":"2024-09-13T14:55:33.509795+0000","last_fullsized":"2024-09-13T14:55:33.509795+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"460'5725","reported_seq":5744,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:37.706639+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:55:37.706639+0000","last_peered":"2024-09-13T14:55:37.706639+0000","last_clean":"2024-09-13T14:55:37.706639+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:55:37.706639+0000","last_undegraded":"2024-09-13T14:55:37.706639+0000","last_fullsized":"2024-09-13T14:55:37.706639+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375193600,"num_objects":5725,"num_object_clones":0,"num_object_copies":17175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":649,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510873+0000","last_change":"2024-09-13T14:40:44.703623+0000","last_active":"2024-09-13T14:55:33.510873+0000","last_peered":"2024-09-13T14:55:33.510873+0000","last_clean":"2024-09-13T14:55:33.510873+0000","last_became_active":"2024-09-13T14:40:44.702793+0000","last_became_peered":"2024-09-13T14:40:44.702793+0000","last_unstale":"2024-09-13T14:55:33.510873+0000","last_undegraded":"2024-09-13T14:55:33.510873+0000","last_fullsized":"2024-09-13T14:55:33.510873+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:40.515368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:40.515368+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:04:19.836641+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":99,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.502592+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:55:33.502592+0000","last_peered":"2024-09-13T14:55:33.502592+0000","last_clean":"2024-09-13T14:55:33.502592+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:55:33.502592+0000","last_undegraded":"2024-09-13T14:55:33.502592+0000","last_fullsized":"2024-09-13T14:55:33.502592+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":878,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513845+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:55:19.513845+0000","last_peered":"2024-09-13T14:55:19.513845+0000","last_clean":"2024-09-13T14:55:19.513845+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:55:19.513845+0000","last_undegraded":"2024-09-13T14:55:19.513845+0000","last_fullsized":"2024-09-13T14:55:19.513845+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":172,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517305+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:55:19.517305+0000","last_peered":"2024-09-13T14:55:19.517305+0000","last_clean":"2024-09-13T14:55:19.517305+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:55:19.517305+0000","last_undegraded":"2024-09-13T14:55:19.517305+0000","last_fullsized":"2024-09-13T14:55:19.517305+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517589+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:55:19.517589+0000","last_peered":"2024-09-13T14:55:19.517589+0000","last_clean":"2024-09-13T14:55:19.517589+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:55:19.517589+0000","last_undegraded":"2024-09-13T14:55:19.517589+0000","last_fullsized":"2024-09-13T14:55:19.517589+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":168,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.518913+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:55:19.518913+0000","last_peered":"2024-09-13T14:55:19.518913+0000","last_clean":"2024-09-13T14:55:19.518913+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:55:19.518913+0000","last_undegraded":"2024-09-13T14:55:19.518913+0000","last_fullsized":"2024-09-13T14:55:19.518913+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":747,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.508769+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:55:33.508769+0000","last_peered":"2024-09-13T14:55:33.508769+0000","last_clean":"2024-09-13T14:55:33.508769+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:55:33.508769+0000","last_undegraded":"2024-09-13T14:55:33.508769+0000","last_fullsized":"2024-09-13T14:55:33.508769+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":459,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.522141+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:55:19.522141+0000","last_peered":"2024-09-13T14:55:19.522141+0000","last_clean":"2024-09-13T14:55:19.522141+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:55:19.522141+0000","last_undegraded":"2024-09-13T14:55:19.522141+0000","last_fullsized":"2024-09-13T14:55:19.522141+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":425,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.510013+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:55:19.510013+0000","last_peered":"2024-09-13T14:55:19.510013+0000","last_clean":"2024-09-13T14:55:19.510013+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:55:19.510013+0000","last_undegraded":"2024-09-13T14:55:19.510013+0000","last_fullsized":"2024-09-13T14:55:19.510013+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":752,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510181+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:55:33.510181+0000","last_peered":"2024-09-13T14:55:33.510181+0000","last_clean":"2024-09-13T14:55:33.510181+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:55:33.510181+0000","last_undegraded":"2024-09-13T14:55:33.510181+0000","last_fullsized":"2024-09-13T14:55:33.510181+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.501797+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:55:33.501797+0000","last_peered":"2024-09-13T14:55:33.501797+0000","last_clean":"2024-09-13T14:55:33.501797+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:55:33.501797+0000","last_undegraded":"2024-09-13T14:55:33.501797+0000","last_fullsized":"2024-09-13T14:55:33.501797+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":808,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509712+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:55:33.509712+0000","last_peered":"2024-09-13T14:55:33.509712+0000","last_clean":"2024-09-13T14:55:33.509712+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:55:33.509712+0000","last_undegraded":"2024-09-13T14:55:33.509712+0000","last_fullsized":"2024-09-13T14:55:33.509712+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":457,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.513783+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:55:19.513783+0000","last_peered":"2024-09-13T14:55:19.513783+0000","last_clean":"2024-09-13T14:55:19.513783+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:55:19.513783+0000","last_undegraded":"2024-09-13T14:55:19.513783+0000","last_fullsized":"2024-09-13T14:55:19.513783+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":840,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509910+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:55:33.509910+0000","last_peered":"2024-09-13T14:55:33.509910+0000","last_clean":"2024-09-13T14:55:33.509910+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:55:33.509910+0000","last_undegraded":"2024-09-13T14:55:33.509910+0000","last_fullsized":"2024-09-13T14:55:33.509910+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.504067+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:55:33.504067+0000","last_peered":"2024-09-13T14:55:33.504067+0000","last_clean":"2024-09-13T14:55:33.504067+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:55:33.504067+0000","last_undegraded":"2024-09-13T14:55:33.504067+0000","last_fullsized":"2024-09-13T14:55:33.504067+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":772,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.508552+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:55:33.508552+0000","last_peered":"2024-09-13T14:55:33.508552+0000","last_clean":"2024-09-13T14:55:33.508552+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:55:33.508552+0000","last_undegraded":"2024-09-13T14:55:33.508552+0000","last_fullsized":"2024-09-13T14:55:33.508552+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":821,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.513657+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:55:33.513657+0000","last_peered":"2024-09-13T14:55:33.513657+0000","last_clean":"2024-09-13T14:55:33.513657+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:55:33.513657+0000","last_undegraded":"2024-09-13T14:55:33.513657+0000","last_fullsized":"2024-09-13T14:55:33.513657+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":157,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517440+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:55:19.517440+0000","last_peered":"2024-09-13T14:55:19.517440+0000","last_clean":"2024-09-13T14:55:19.517440+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:55:19.517440+0000","last_undegraded":"2024-09-13T14:55:19.517440+0000","last_fullsized":"2024-09-13T14:55:19.517440+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":718,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.509341+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:55:33.509341+0000","last_peered":"2024-09-13T14:55:33.509341+0000","last_clean":"2024-09-13T14:55:33.509341+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:55:33.509341+0000","last_undegraded":"2024-09-13T14:55:33.509341+0000","last_fullsized":"2024-09-13T14:55:33.509341+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":94,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.503328+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:55:33.503328+0000","last_peered":"2024-09-13T14:55:33.503328+0000","last_clean":"2024-09-13T14:55:33.503328+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:55:33.503328+0000","last_undegraded":"2024-09-13T14:55:33.503328+0000","last_fullsized":"2024-09-13T14:55:33.503328+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517520+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:55:19.517520+0000","last_peered":"2024-09-13T14:55:19.517520+0000","last_clean":"2024-09-13T14:55:19.517520+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:55:19.517520+0000","last_undegraded":"2024-09-13T14:55:19.517520+0000","last_fullsized":"2024-09-13T14:55:19.517520+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":425,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509479+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:55:19.509479+0000","last_peered":"2024-09-13T14:55:19.509479+0000","last_clean":"2024-09-13T14:55:19.509479+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:55:19.509479+0000","last_undegraded":"2024-09-13T14:55:19.509479+0000","last_fullsized":"2024-09-13T14:55:19.509479+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517358+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:55:19.517358+0000","last_peered":"2024-09-13T14:55:19.517358+0000","last_clean":"2024-09-13T14:55:19.517358+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:55:19.517358+0000","last_undegraded":"2024-09-13T14:55:19.517358+0000","last_fullsized":"2024-09-13T14:55:19.517358+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":391,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.509851+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:55:19.509851+0000","last_peered":"2024-09-13T14:55:19.509851+0000","last_clean":"2024-09-13T14:55:19.509851+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:55:19.509851+0000","last_undegraded":"2024-09-13T14:55:19.509851+0000","last_fullsized":"2024-09-13T14:55:19.509851+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":832,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510517+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:55:33.510517+0000","last_peered":"2024-09-13T14:55:33.510517+0000","last_clean":"2024-09-13T14:55:33.510517+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:55:33.510517+0000","last_undegraded":"2024-09-13T14:55:33.510517+0000","last_fullsized":"2024-09-13T14:55:33.510517+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":805,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.510868+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:55:33.510868+0000","last_peered":"2024-09-13T14:55:33.510868+0000","last_clean":"2024-09-13T14:55:33.510868+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:55:33.510868+0000","last_undegraded":"2024-09-13T14:55:33.510868+0000","last_fullsized":"2024-09-13T14:55:33.510868+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":158,"reported_epoch":459,"state":"active+clean","last_fresh":"2024-09-13T14:55:19.517157+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:55:19.517157+0000","last_peered":"2024-09-13T14:55:19.517157+0000","last_clean":"2024-09-13T14:55:19.517157+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:55:19.517157+0000","last_undegraded":"2024-09-13T14:55:19.517157+0000","last_fullsized":"2024-09-13T14:55:19.517157+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":101,"reported_epoch":460,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.501743+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:55:33.501743+0000","last_peered":"2024-09-13T14:55:33.501743+0000","last_clean":"2024-09-13T14:55:33.501743+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:55:33.501743+0000","last_undegraded":"2024-09-13T14:55:33.501743+0000","last_fullsized":"2024-09-13T14:55:33.501743+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5959319552,"num_objects":90932,"num_object_clones":0,"num_object_copies":272796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90932,"num_write_kb":5819648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17828151296,"data_stored":17828151296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90932,"ondisk_log_size":90932,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398034,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2360068,"kb_used_data":1468220,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":91385084,"statfs":{"total":95995035648,"available":93578326016,"internally_reserved":0,"allocated":1503457280,"data_stored":1501008836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594262,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3465524,"kb_used_data":2562732,"kb_used_omap":1,"kb_used_meta":902782,"kb_avail":90279628,"statfs":{"total":95995035648,"available":92446339072,"internally_reserved":0,"allocated":2624237568,"data_stored":2621804806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":924449112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665716,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2752688,"kb_used_data":1826664,"kb_used_omap":9,"kb_used_meta":926006,"kb_avail":90992464,"statfs":{"total":95995035648,"available":93176283136,"internally_reserved":0,"allocated":1870503936,"data_stored":1868071577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":948230344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986718,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3813312,"kb_used_data":2909304,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":89931840,"statfs":{"total":95995035648,"available":92090204160,"internally_reserved":0,"allocated":2979127296,"data_stored":2976682878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697278,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2909040,"kb_used_data":1855656,"kb_used_omap":4,"kb_used_meta":1053371,"kb_avail":90836112,"statfs":{"total":95995035648,"available":93016178688,"internally_reserved":0,"allocated":1900191744,"data_stored":1897759466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1078651979},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860804,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2919096,"kb_used_data":2109164,"kb_used_omap":8,"kb_used_meta":809911,"kb_avail":90826056,"statfs":{"total":95995035648,"available":93005881344,"internally_reserved":0,"allocated":2159783936,"data_stored":2157348076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":829349708},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020475,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702572,"kb_used_data":1069668,"kb_used_omap":9,"kb_used_meta":632886,"kb_avail":92042580,"statfs":{"total":95995035648,"available":94251601920,"internally_reserved":0,"allocated":1095340032,"data_stored":1092911227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":648076101},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187866,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4490860,"kb_used_data":3609828,"kb_used_omap":7,"kb_used_meta":881016,"kb_avail":89254292,"statfs":{"total":95995035648,"available":91396395008,"internally_reserved":0,"allocated":3696463872,"data_stored":3694035333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":902160715},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3697410048,"data_stored":3697410048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1093730304,"data_stored":1093730304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2179727360,"data_stored":2179727360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1884225536,"data_stored":1884225536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2976120832,"data_stored":2976120832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1880227840,"data_stored":1880227840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2618425344,"data_stored":2618425344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1498284032,"data_stored":1498284032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:41.778 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:55:41.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-09-13T14:55:41.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:41 smithi017 ceph-mon[103001]: from='client.47030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:41.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:41 smithi017 ceph-mon[107556]: from='client.47030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:41.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:41.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.999 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:41 smithi189 ceph-mon[90992]: from='client.47030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:42.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 95605 95589 101.24 70.875 0.0216 0.00987214 2024-09-13T14:55:42.030 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:42.036 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:42.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:42.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:42.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:42.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.422 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:42.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:42.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[103001]: pgmap v1506: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:42.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:42.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[107556]: pgmap v1506: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:42.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:42.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:42.808 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:42.840 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:42.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:55:43.002155+0000 min lat: 0.00143583 max lat: 0.474113 avg lat: 0.00992781 2024-09-13T14:55:43.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:55:43.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 96680 96664 100.672 67.1875 0.00955807 0.00992781 2024-09-13T14:55:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:42 smithi189 ceph-mon[90992]: pgmap v1506: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-13T14:55:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:55:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:43.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.223 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.257 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:43.359 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:43.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:43.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:43.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:43.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.564 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:55:43.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:43.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:43.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.882 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1081116349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.882 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.882 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1081116349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.882 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:44.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 97813 97797 100.183 70.8125 0.0057243 0.00997621 2024-09-13T14:55:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1081116349' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.079 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:44.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:44.217 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.388 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.490 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:44.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:44.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:44.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:44.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:44.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:44.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[103001]: pgmap v1507: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:44.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[103001]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:55:44.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[107556]: pgmap v1507: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:44.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:44 smithi017 ceph-mon[107556]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:55:44.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:44.930 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:55:44.930 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:44.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:44.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:55:44.971 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:45.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 98962 98946 99.725 71.8125 0.00931058 0.0100224 2024-09-13T14:55:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:44 smithi189 ceph-mon[90992]: pgmap v1507: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:44 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:44 smithi189 ceph-mon[90992]: osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:55:45.010 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:55:45.010 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:45.049 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:55:45.050 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:45.054 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.079 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:55:45.079 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:45.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:55:45.114 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:45.152 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:55:45.152 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:45.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.192 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:55:45.193 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:45.225 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.327 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:45.361 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:45.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.498 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.600 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.633 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.735 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.766 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.868 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.909 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:45.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3576982612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3576982612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:46.002 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 100086 100070 99.2571 70.25 0.0107213 0.0100699 2024-09-13T14:55:46.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3576982612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:46.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:46.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:46.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:46.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:46.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:46.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:46.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:46.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:46.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:46.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:46.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:55:46.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:46.744 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:46.846 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:46.869 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:46 smithi017 ceph-mon[103001]: pgmap v1509: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:46.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:46 smithi017 ceph-mon[107556]: pgmap v1509: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:46.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:46.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:46.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:46.978 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:47.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:47.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 101167 101151 98.7619 67.5625 0.0331065 0.0101203 2024-09-13T14:55:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:47.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:46 smithi189 ceph-mon[90992]: pgmap v1509: 99 pgs: 99 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-13T14:55:47.108 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:47.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:47.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:47.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:47.402 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:47] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.51.0" 2024-09-13T14:55:47.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:47.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:47.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:47.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:47.662 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.693 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:47.795 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.822 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:47.822 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:47.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:47.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:47.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.004 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 102191 102175 98.227 64 0.00911178 0.0101756 2024-09-13T14:55:48.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:48.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:48.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:48.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:48.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:48.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:48.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.709 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:48.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:48.810 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:48.846 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:48.934 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:48 smithi017 ceph-mon[103001]: pgmap v1510: 99 pgs: 99 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:48.934 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:48 smithi017 ceph-mon[107556]: pgmap v1510: 99 pgs: 99 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:48.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:49.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:49.003 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 103258 103242 97.749 66.6875 0.0263182 0.0102246 2024-09-13T14:55:49.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:48 smithi189 ceph-mon[90992]: pgmap v1510: 99 pgs: 99 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:49.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.234 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:49.272 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:49.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:49.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:49.507 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:49.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:49.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:55:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:49.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:49.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.743 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.845 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.889 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:49.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.991 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:50.003 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 104295 104279 97.2574 64.8125 0.0208485 0.0102771 2024-09-13T14:55:50.033 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:50.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:50.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:50.195 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:50.277 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:50.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:50.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:50.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:55:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:55:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:50.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:50 smithi017 ceph-mon[103001]: pgmap v1511: 99 pgs: 99 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:50 smithi017 ceph-mon[107556]: pgmap v1511: 99 pgs: 99 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:50.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:50.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:50.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:50.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:50.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.005 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 105301 105285 96.749 62.875 0.0112933 0.010331 2024-09-13T14:55:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:50 smithi189 ceph-mon[90992]: pgmap v1511: 99 pgs: 99 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:51.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:51.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:51.265 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.298 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:51.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:51.399 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.435 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.577 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 21 2024-09-13T14:55:51.584 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:51.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:51 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:51.782 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:51 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:51.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:51.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:52.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.005 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 106397 106381 96.3395 68.5 0.0124052 0.010375 2024-09-13T14:55:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:51 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-13T14:55:52.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:52.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:52.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:52.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.377 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.419 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:52.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:52.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:52.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:52.663 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.698 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:52.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:52.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:52.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:52.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[103001]: pgmap v1512: 99 pgs: 99 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[103001]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:55:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[107556]: pgmap v1512: 99 pgs: 99 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:52 smithi017 ceph-mon[107556]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:55:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:53.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.005 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 107482 107466 95.9317 67.8125 0.0179074 0.0104191 2024-09-13T14:55:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:52 smithi189 ceph-mon[90992]: pgmap v1512: 99 pgs: 99 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:55:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1459942620' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-13T14:55:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:52 smithi189 ceph-mon[90992]: osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:55:53.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:53.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:53.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:53.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:53.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:53.543 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:53.560 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:53.662 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.703 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:53.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:53.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:53.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:53.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:53.837 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:53.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:54.005 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 108544 108528 95.5154 66.375 0.00870961 0.0104648 2024-09-13T14:55:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:54.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:54.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:54.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:54.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:54.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:54.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[103001]: pgmap v1514: 99 pgs: 99 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[103001]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:55:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[107556]: pgmap v1514: 99 pgs: 99 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:54 smithi017 ceph-mon[107556]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:55:54.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:54.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:54.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:54 smithi189 ceph-mon[90992]: pgmap v1514: 99 pgs: 99 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:55:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:54 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-13T14:55:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:54 smithi189 ceph-mon[90992]: osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:55:55.008 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 109597 109581 95.1022 65.8125 0.0162083 0.0105105 2024-09-13T14:55:55.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:55.105 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:55.206 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:55.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.487 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:55.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:55.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:55.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:55:55.718 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:55.754 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:55:55.754 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:55.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:55:55.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[103001]: osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[103001]: 1.14 scrub starts 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[107556]: osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[107556]: 1.14 scrub starts 2024-09-13T14:55:55.787 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:55 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:55:55.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:55:55.826 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:55.856 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:55:55.857 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:55.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:55:55.889 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:55.911 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:55.920 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:55:55.920 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:55.944 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:55.958 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:55:55.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:55 smithi189 ceph-mon[90992]: osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:55:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:55 smithi189 ceph-mon[90992]: 1.14 scrub starts 2024-09-13T14:55:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:55 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:55:56.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 110650 110634 94.7009 65.8125 0.0150583 0.0105549 2024-09-13T14:55:56.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:56.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:56.218 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:56.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:56.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:56.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:56.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:56.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:56.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:56.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:56.782 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[103001]: pgmap v1517: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1206567900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[103001]: 1.13 scrub starts 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[107556]: pgmap v1517: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1206567900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:55:56.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[107556]: 1.13 scrub starts 2024-09-13T14:55:56.849 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:56.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:56.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:56 smithi189 ceph-mon[90992]: pgmap v1517: 109 pgs: 1 peering, 10 unknown, 98 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-09-13T14:55:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1206567900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:56 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:55:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:56 smithi189 ceph-mon[90992]: 1.13 scrub starts 2024-09-13T14:55:57.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:57.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 111470 111454 94.1137 51.25 0.00428902 0.010615 2024-09-13T14:55:57.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:57.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:57.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:57.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.154 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:55:57.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:57.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:57.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:57.410 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:57.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:57.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:55:57] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.51.0" 2024-09-13T14:55:57.545 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:55:57.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:55:57.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:57.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:57.732 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:57.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:57.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:57.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:57.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:55:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: 1.c scrub starts 2024-09-13T14:55:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: 1.c scrub ok 2024-09-13T14:55:57.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:57.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: 1.b scrub starts 2024-09-13T14:55:57.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[107556]: 1.b scrub ok 2024-09-13T14:55:57.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: 1.c scrub starts 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: 1.c scrub ok 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:57.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: 1.b scrub starts 2024-09-13T14:55:57.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:57 smithi017 ceph-mon[103001]: 1.b scrub ok 2024-09-13T14:55:57.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/578700021' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: 1.c scrub starts 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: 1.c scrub ok 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: 1.b scrub starts 2024-09-13T14:55:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:57 smithi189 ceph-mon[90992]: 1.b scrub ok 2024-09-13T14:55:58.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 112049 112033 93.3414 36.1875 0.165945 0.0106915 2024-09-13T14:55:58.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:58.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:58.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:58.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:58.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:58 smithi017 ceph-mon[103001]: pgmap v1519: 109 pgs: 1 active+clean+scrubbing, 1 peering, 8 unknown, 99 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:58 smithi017 ceph-mon[107556]: pgmap v1519: 109 pgs: 1 active+clean+scrubbing, 1 peering, 8 unknown, 99 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:58.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:58.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:58.927 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.966 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:58 smithi189 ceph-mon[90992]: pgmap v1519: 109 pgs: 1 active+clean+scrubbing, 1 peering, 8 unknown, 99 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:55:59.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 113022 113006 92.9134 60.8125 0.0124245 0.0107577 2024-09-13T14:55:59.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:59.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.201 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.236 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:59.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.622 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:59.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:59.746 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.784 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:55:59.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:59.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:59.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:55:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[103001]: 1.0 scrub starts 2024-09-13T14:55:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[103001]: 1.0 scrub ok 2024-09-13T14:55:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:55:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[107556]: 1.0 scrub starts 2024-09-13T14:55:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[107556]: 1.0 scrub ok 2024-09-13T14:55:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:55:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:56:00.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:59 smithi189 ceph-mon[90992]: 1.0 scrub starts 2024-09-13T14:56:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:59 smithi189 ceph-mon[90992]: 1.0 scrub ok 2024-09-13T14:56:00.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:55:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:56:00.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 114044 114028 92.5362 63.875 0.0059769 0.0108015 2024-09-13T14:56:00.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.167 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:00.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:00.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:00.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:00.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:00.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:00.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:00.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:00.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:00.703 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.734 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:00.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.2 scrub starts 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.2 scrub ok 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: pgmap v1520: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.7 scrub starts 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.7 scrub ok 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.4 scrub starts 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.4 scrub ok 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.3 scrub starts 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[103001]: 1.3 scrub ok 2024-09-13T14:56:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.2 scrub starts 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.2 scrub ok 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: pgmap v1520: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.7 scrub starts 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.7 scrub ok 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.4 scrub starts 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.4 scrub ok 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.3 scrub starts 2024-09-13T14:56:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:00 smithi017 ceph-mon[107556]: 1.3 scrub ok 2024-09-13T14:56:00.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:01.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 114941 114925 92.0686 56.0625 0.0196329 0.0108566 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.2 scrub starts 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.2 scrub ok 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: pgmap v1520: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.7 scrub starts 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.7 scrub ok 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.4 scrub starts 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.4 scrub ok 2024-09-13T14:56:01.008 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.3 scrub starts 2024-09-13T14:56:01.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:00 smithi189 ceph-mon[90992]: 1.3 scrub ok 2024-09-13T14:56:01.010 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:01.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:01.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:01.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.305 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:01.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:01.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:01.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:01.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:01.577 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:01.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:01.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.1 scrub starts 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.1 scrub ok 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.6 scrub starts 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.6 scrub ok 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.5 scrub ok 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.13 scrub starts 2024-09-13T14:56:01.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:01 smithi189 ceph-mon[90992]: 1.13 scrub ok 2024-09-13T14:56:01.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:01.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.1 scrub starts 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.1 scrub ok 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.6 scrub starts 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.6 scrub ok 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.5 scrub ok 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.13 scrub starts 2024-09-13T14:56:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[103001]: 1.13 scrub ok 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.1 scrub starts 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.1 scrub ok 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.6 scrub starts 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.6 scrub ok 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.5 scrub ok 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.13 scrub starts 2024-09-13T14:56:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:01 smithi017 ceph-mon[107556]: 1.13 scrub ok 2024-09-13T14:56:01.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:02.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 115934 115918 91.6887 62.0625 0.0152027 0.0109013 2024-09-13T14:56:02.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:02.113 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:02.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:02.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:02.340 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:56:02.341 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:56:02.341 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:56:02.341 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:56:02.341 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:56:02.387 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:02.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:02.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:02.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:02.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:02.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.833 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:02.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.11 scrub starts 2024-09-13T14:56:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.11 scrub ok 2024-09-13T14:56:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.d scrub starts 2024-09-13T14:56:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.d scrub ok 2024-09-13T14:56:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: pgmap v1523: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-09-13T14:56:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.12 scrub starts 2024-09-13T14:56:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[107556]: 1.12 scrub ok 2024-09-13T14:56:02.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.11 scrub starts 2024-09-13T14:56:02.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.11 scrub ok 2024-09-13T14:56:02.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.d scrub starts 2024-09-13T14:56:02.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.d scrub ok 2024-09-13T14:56:02.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: pgmap v1523: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-09-13T14:56:02.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.12 scrub starts 2024-09-13T14:56:02.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:02 smithi017 ceph-mon[103001]: 1.12 scrub ok 2024-09-13T14:56:02.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.11 scrub starts 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.11 scrub ok 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.d scrub starts 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.d scrub ok 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: pgmap v1523: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.12 scrub starts 2024-09-13T14:56:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:02 smithi189 ceph-mon[90992]: 1.12 scrub ok 2024-09-13T14:56:03.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-13T14:56:03.006666+0000 min lat: 0.00143583 max lat: 0.474113 avg lat: 0.010946 2024-09-13T14:56:03.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:56:03.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 116924 116908 91.316 61.875 0.0227415 0.010946 2024-09-13T14:56:03.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:03.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:03.212 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.245 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.384 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:03.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:03.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:03.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:03.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:03.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:03.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:03.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:56:03.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:56:03.753 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:03.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:03 smithi017 ceph-mon[107556]: 1.f scrub starts 2024-09-13T14:56:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:03 smithi017 ceph-mon[107556]: 1.f scrub ok 2024-09-13T14:56:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:03 smithi017 ceph-mon[103001]: 1.f scrub starts 2024-09-13T14:56:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:03 smithi017 ceph-mon[103001]: 1.f scrub ok 2024-09-13T14:56:03.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:03.977 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:04.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:03 smithi189 ceph-mon[90992]: 1.f scrub starts 2024-09-13T14:56:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:03 smithi189 ceph-mon[90992]: 1.f scrub ok 2024-09-13T14:56:04.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 117942 117926 90.9741 63.625 0.0221163 0.0109877 2024-09-13T14:56:04.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:04.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:04.450 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:04.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:04.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:04.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:04.673 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:04.682 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:04.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:04.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:04.895 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:04.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:04.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[103001]: pgmap v1524: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 84 MiB/s wr, 1.34k op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:04.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[103001]: 1.10 scrub starts 2024-09-13T14:56:04.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[103001]: 1.10 scrub ok 2024-09-13T14:56:04.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[107556]: pgmap v1524: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 84 MiB/s wr, 1.34k op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:04.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[107556]: 1.10 scrub starts 2024-09-13T14:56:04.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:04 smithi017 ceph-mon[107556]: 1.10 scrub ok 2024-09-13T14:56:04.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:04 smithi189 ceph-mon[90992]: pgmap v1524: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 84 MiB/s wr, 1.34k op/s; 2.6 KiB/s, 0 objects/s recovering 2024-09-13T14:56:05.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:04 smithi189 ceph-mon[90992]: 1.10 scrub starts 2024-09-13T14:56:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:04 smithi189 ceph-mon[90992]: 1.10 scrub ok 2024-09-13T14:56:05.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 118902 118886 90.5963 60 0.010736 0.0110322 2024-09-13T14:56:05.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:05.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.170 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:05.272 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:05.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.476 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:05.578 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.620 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:05.722 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:05.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.911 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:05.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:05 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:05.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:05 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:06.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:05 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:06.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 119832 119816 90.2051 58.125 0.0213365 0.011081 2024-09-13T14:56:06.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.049 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.150 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.183 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.285 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:06.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:06.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:06.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:06.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:06.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:06.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:06.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.861 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.893 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:06.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:06 smithi189 ceph-mon[90992]: pgmap v1525: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:06 smithi189 ceph-mon[90992]: 1.e scrub starts 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:06 smithi189 ceph-mon[90992]: 1.e scrub ok 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[103001]: pgmap v1525: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[103001]: 1.e scrub starts 2024-09-13T14:56:07.006 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[103001]: 1.e scrub ok 2024-09-13T14:56:07.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[107556]: pgmap v1525: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:07.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[107556]: 1.e scrub starts 2024-09-13T14:56:07.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:06 smithi017 ceph-mon[107556]: 1.e scrub ok 2024-09-13T14:56:07.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 120811 120795 89.8595 61.1875 0.0204127 0.0111227 2024-09-13T14:56:07.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:07.138 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:07.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:07.274 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:07.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:07.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:07.455 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:07.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:07.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:07.648 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:07] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.51.0" 2024-09-13T14:56:07.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:07.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:07.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:07.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:07.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 121790 121774 89.5221 61.1875 0.00834806 0.0111649 2024-09-13T14:56:08.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.068 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:08.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:08.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:08.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:08.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:08.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:08.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:08.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:08.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[107556]: 1.14 scrub starts 2024-09-13T14:56:08.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[107556]: 1.14 scrub ok 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[107556]: pgmap v1526: 109 pgs: 109 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 46 MiB/s wr, 737 op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[103001]: 1.14 scrub starts 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[103001]: 1.14 scrub ok 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[103001]: pgmap v1526: 109 pgs: 109 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 46 MiB/s wr, 737 op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:08.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:08 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:08.948 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:08 smithi189 ceph-mon[90992]: 1.14 scrub starts 2024-09-13T14:56:08.948 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:08 smithi189 ceph-mon[90992]: 1.14 scrub ok 2024-09-13T14:56:08.949 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:08 smithi189 ceph-mon[90992]: pgmap v1526: 109 pgs: 109 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 2.9 KiB/s rd, 46 MiB/s wr, 737 op/s; 42 KiB/s, 0 objects/s recovering 2024-09-13T14:56:08.949 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:08 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:08.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.010 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 122715 122699 89.1508 57.8125 0.016466 0.0112109 2024-09-13T14:56:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:09.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:09.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:09.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:09.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:09.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:09.434 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:09.471 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:09.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:09.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:09.632 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:56:09.632 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:56:09.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:09.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:09.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:09.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:09.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4209547587' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2538155416' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4209547587' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2538155416' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:09.976 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4209547587' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2538155416' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:56:10.010 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 15 123437 123422 88.6455 45.1875 0.0139134 0.0112761 2024-09-13T14:56:10.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:10.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:10.253 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:10.392 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.433 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:10.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:10.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:10.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:10.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:10.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:10.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: pgmap v1527: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[103001]: osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: pgmap v1527: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:56:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:10.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:56:10.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:10 smithi017 ceph-mon[107556]: osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:56:10.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:10.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: pgmap v1527: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:56:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:10 smithi189 ceph-mon[90992]: osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:56:11.010 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 124416 124400 88.3327 61.125 0.0164532 0.011316 2024-09-13T14:56:11.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:11.227 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:11.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:11.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.542 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:11.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:11.678 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:11 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:11 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:11.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:11.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:11.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:11.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:11 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:12.010 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 125392 125376 88.0255 61 0.0149218 0.0113556 2024-09-13T14:56:12.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:12.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:12.395 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:12.497 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:12.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:12.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:12.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[107556]: pgmap v1529: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[103001]: pgmap v1529: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:12.737 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:12.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:12.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:12.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:12.956 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:12.981 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:56:12.981 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:56:12.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:56:12.981 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 302.94667368199998, 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32587973199999998, 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:12.982 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786731+0000", 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2778000000000001e-05 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:12.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786747+0000", 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6589e-05 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786753+0000", 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7810000000000002e-06 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792834+0000", 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060810639999999997 2024-09-13T14:56:12.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792996+0000", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016137599999999999 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793158+0000", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162034 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797490+0000", 2024-09-13T14:56:12.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043325070000000002 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805581+0000", 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080903590000000001 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112577+0000", 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30699643500000001 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112587+0000", 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031e-05 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95184 9.2 e382/376 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head v 382'6052, mlcod=382'6047)", 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 302.93305622499997, 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35791878900000001, 2024-09-13T14:56:12.988 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800326+0000", 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0500000000000002e-07 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800353+0000", 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6665e-05 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800365+0000", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.205e-05 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800374+0000", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801552+0000", 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011782439999999999 2024-09-13T14:56:12.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801573+0000", 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0863999999999999e-05 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.158244+0000", 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35667120099999999 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95182 9.0 9:0107e64f:::benchmark_data_smithi017_339564_object95181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 302.81467247, 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.87911086900000002, 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:12.993 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95182 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918711+0000", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5579999999999999e-06 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918751+0000", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9842999999999998e-05 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918774+0000", 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2779999999999999e-05 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918785+0000", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112614+0000", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19382977600000001 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112623+0000", 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0620000000000007e-06 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.796000+0000", 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68337630999999999 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.797820+0000", 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00182003 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.997 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 302.814575601, 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88068354999999998, 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:12.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5799999999999999e-07 2024-09-13T14:56:12.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919323+0000", 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051669300000000004 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919336+0000", 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3485999999999999e-05 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919344+0000", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112646+0000", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19330140800000001 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112652+0000", 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6499999999999999e-06 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799403+0000", 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68675042600000002 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799489+0000", 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6649000000000003e-05 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.002 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99743 9.10 9:09e78ae4:::benchmark_data_smithi017_339564_object99742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82170356099999, 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2203931610000001, 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99743 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.003 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911679+0000", 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3689999999999999e-06 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911728+0000", 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.922e-05 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911741+0000", 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2315e-05 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911746+0000", 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8080000000000001e-06 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911801+0000", 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4157999999999999e-05 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911808+0000", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6739999999999997e-06 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102095+0000", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1902865440000001 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102270+0000", 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017495 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102476+0000", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000205787 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109826+0000", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073506869999999998 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109842+0000", 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.554e-05 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115907+0000", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060650799999999996 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129977+0000", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014069813 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.130088+0000", 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111799 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132071+0000", 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019824170000000002 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99758 9.10 9:0b28f730:::benchmark_data_smithi017_339564_object99757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82161930900003, 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2224807790000001, 2024-09-13T14:56:13.010 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99758 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911763+0000", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2e-07 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911826+0000", 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3499999999999999e-05 2024-09-13T14:56:13.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911834+0000", 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9570000000000002e-06 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911838+0000", 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0049999999999998e-06 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911869+0000", 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0925e-05 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911873+0000", 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9539999999999998e-06 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105698+0000", 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.193825261 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105855+0000", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015642 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.106033+0000", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017866900000000001 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109848+0000", 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038150580000000001 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121439+0000", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01159059 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132121+0000", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682087999999999 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132209+0000", 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7552000000000005e-05 2024-09-13T14:56:13.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132324+0000", 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115606 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134243+0000", 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001918674 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.017 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99769 9.10 9:081d7eac:::benchmark_data_smithi017_339564_object99768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82153248999998, 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.223936653, 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99769 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8800000000000003e-07 2024-09-13T14:56:13.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911912+0000", 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2597999999999999e-05 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911920+0000", 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4309999999999994e-06 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911924+0000", 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1389999999999997e-06 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911955+0000", 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0182e-05 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911958+0000", 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6330000000000002e-06 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.107964+0000", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1960057749999999 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108122+0000", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015787400000000001 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108353+0000", 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023077800000000001 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115855+0000", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075024380000000002 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121484+0000", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056290380000000003 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132167+0000", 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682957 2024-09-13T14:56:13.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134290+0000", 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021226890000000001 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134356+0000", 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5739000000000002e-05 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135785+0000", 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014298939999999999 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99779 9.10 9:0b953e14:::benchmark_data_smithi017_339564_object99778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82144617400002, 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.225690441, 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.025 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99779 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911936+0000", 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911991+0000", 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5569999999999998e-05 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911999+0000", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5089999999999996e-06 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912003+0000", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9319999999999998e-06 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912031+0000", 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8237000000000001e-05 2024-09-13T14:56:13.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912034+0000", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.444e-06 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110235+0000", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.198200776 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110443+0000", 2024-09-13T14:56:13.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000207517 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110617+0000", 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017471500000000001 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115921+0000", 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053037340000000001 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135838+0000", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019916776000000001 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135894+0000", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6515999999999999e-05 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136027+0000", 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013228 2024-09-13T14:56:13.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136088+0000", 2024-09-13T14:56:13.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1495999999999995e-05 2024-09-13T14:56:13.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137626+0000", 2024-09-13T14:56:13.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015374570000000001 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99792 9.10 9:0afafc95:::benchmark_data_smithi017_339564_object99791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82136795500003, 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2470098199999999, 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.034 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.035 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.035 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.035 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99792 2024-09-13T14:56:13.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912014+0000", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7599999999999997e-07 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912055+0000", 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0843999999999997e-05 2024-09-13T14:56:13.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912062+0000", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3640000000000001e-06 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912066+0000", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5379999999999998e-06 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912091+0000", 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5678999999999999e-05 2024-09-13T14:56:13.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912095+0000", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4240000000000002e-06 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113399+0000", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2013038380000001 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113589+0000", 2024-09-13T14:56:13.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019008499999999999 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113800+0000", 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021092600000000001 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121388+0000", 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007588086 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135939+0000", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014551708 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137677+0000", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001737527 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156695+0000", 2024-09-13T14:56:13.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019018576999999998 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156755+0000", 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0028000000000002e-05 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159023+0000", 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00226772 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.042 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99812 9.10 9:08133f40:::benchmark_data_smithi017_339564_object99811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82130799100003, 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2497001430000001, 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99812 2024-09-13T14:56:13.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912074+0000", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7999999999999996e-07 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912114+0000", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9949e-05 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912121+0000", 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0820000000000001e-06 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912124+0000", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4309999999999998e-06 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912148+0000", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3368999999999999e-05 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912151+0000", 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.45e-06 2024-09-13T14:56:13.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115948+0000", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.203796654 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116167+0000", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021876999999999999 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116334+0000", 2024-09-13T14:56:13.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000167794 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121395+0000", 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050604029999999998 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135984+0000", 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014589289 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137719+0000", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001734684 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159936+0000", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022217263000000001 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.160008+0000", 2024-09-13T14:56:13.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1608999999999998e-05 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161773+0000", 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001765916 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99825 9.10 9:0c0deca2:::benchmark_data_smithi017_339564_object99824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82124930499998, 2024-09-13T14:56:13.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2555997640000001, 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99825 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:56:13.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912171+0000", 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8818e-05 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912178+0000", 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9840000000000004e-06 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912182+0000", 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7809999999999999e-06 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912230+0000", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7647e-05 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912240+0000", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.005e-05 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118734+0000", 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2064939020000001 2024-09-13T14:56:13.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118892+0000", 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158796 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.119058+0000", 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000165336 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129912+0000", 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010854278 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145102+0000", 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015189947000000001 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145183+0000", 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1160000000000002e-05 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165672+0000", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020488431000000001 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165735+0000", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2934000000000004e-05 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.167732+0000", 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00199723 2024-09-13T14:56:13.057 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99867 9.10 9:08792be1:::benchmark_data_smithi017_339564_object99866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82119148999999, 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.258038813, 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99867 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7399999999999998e-07 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912235+0000", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4255999999999998e-05 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912244+0000", 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3999999999999998e-06 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912250+0000", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912279+0000", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8410000000000001e-05 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912283+0000", 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8190000000000002e-06 2024-09-13T14:56:13.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121515+0000", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2092329669999999 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121672+0000", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015640000000000001 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121848+0000", 2024-09-13T14:56:13.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017612800000000001 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129919+0000", 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080710069999999998 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145146+0000", 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015227021 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145227+0000", 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1304000000000006e-05 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168260+0000", 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023032357999999999 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168415+0000", 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015499000000000001 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170229+0000", 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018139320000000001 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99888 9.10 9:09cf0b05:::benchmark_data_smithi017_339564_object99887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82112013199998, 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.260453754, 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.065 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99888 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912262+0000", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912307+0000", 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5405999999999999e-05 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912314+0000", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3220000000000004e-06 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912318+0000", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7019999999999999e-06 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912344+0000", 2024-09-13T14:56:13.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.614e-05 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912348+0000", 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.597e-06 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.123904+0000", 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.211556031 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124058+0000", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153933 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124254+0000", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019642300000000001 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140317+0000", 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016063137000000002 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152386+0000", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012068270000000001 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153192+0000", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080682099999999999 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170291+0000", 2024-09-13T14:56:13.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017098248 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170362+0000", 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1415999999999995e-05 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172715+0000", 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002352809 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99894 9.10 9:0e8bfb10:::benchmark_data_smithi017_339564_object99893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82105526200002, 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2902429099999999, 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99894 2024-09-13T14:56:13.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4000000000000002e-07 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912366+0000", 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9403999999999997e-05 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912373+0000", 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0910000000000004e-06 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912377+0000", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6890000000000002e-06 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912402+0000", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5584000000000001e-05 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912406+0000", 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4419999999999998e-06 2024-09-13T14:56:13.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126634+0000", 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.214228753 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126788+0000", 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015391299999999999 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126969+0000", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018079000000000001 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145055+0000", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018085981000000001 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152432+0000", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073765209999999996 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153227+0000", 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079486599999999998 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200627+0000", 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047400281000000002 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200692+0000", 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5309999999999996e-05 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202569+0000", 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001876845 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99941 9.10 9:0dcd111c:::benchmark_data_smithi017_339564_object99940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82099686200002, 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.291876698, 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99941 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912385+0000", 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8599999999999998e-07 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912423+0000", 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8223000000000003e-05 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912430+0000", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8310000000000003e-06 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912434+0000", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6679999999999999e-06 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912459+0000", 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5049999999999999e-05 2024-09-13T14:56:13.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912462+0000", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.439e-06 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137750+0000", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2252876800000001 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137916+0000", 2024-09-13T14:56:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166047 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.138092+0000", 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017649300000000001 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145062+0000", 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069691930000000003 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153258+0000", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081965000000000007 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156056+0000", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002798067 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202632+0000", 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.046576059000000003 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202696+0000", 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4141999999999997e-05 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204261+0000", 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00156482 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.087 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99950 9.10 9:0d57d527:::benchmark_data_smithi017_339564_object99949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82094005699997, 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2935929129999999, 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99950 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912442+0000", 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912480+0000", 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8099999999999998e-05 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912487+0000", 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8990000000000004e-06 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912490+0000", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912526+0000", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5649999999999999e-05 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7969999999999998e-06 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140490+0000", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2279604200000001 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141138+0000", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064823500000000004 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141336+0000", 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019801899999999999 2024-09-13T14:56:13.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145114+0000", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037779839999999999 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159122+0000", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014007400999999999 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.163411+0000", 2024-09-13T14:56:13.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042897459999999997 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204322+0000", 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.040910421000000002 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204554+0000", 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023251700000000001 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206034+0000", 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014799290000000001 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99954 9.10 9:0ddee9ee:::benchmark_data_smithi017_339564_object99953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82088410599999, 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2956523129999999, 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.095 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99954 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912498+0000", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.89e-07 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912517+0000", 2024-09-13T14:56:13.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9406999999999999e-05 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912525+0000", 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3699999999999995e-06 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7469999999999999e-06 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912563+0000", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3584999999999998e-05 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912571+0000", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0949999999999996e-06 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143130+0000", 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.230558984 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143272+0000", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000142399 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143498+0000", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022541600000000001 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145240+0000", 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017423510000000001 2024-09-13T14:56:13.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161922+0000", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016681756999999998 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172781+0000", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010859051 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206100+0000", 2024-09-13T14:56:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033319357000000001 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206306+0000", 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020586899999999999 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208150+0000", 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018434359999999999 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99965 9.10 9:081c4e71:::benchmark_data_smithi017_339564_object99964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82084480499998, 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2972334430000001, 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99965 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0900000000000002e-07 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912588+0000", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1122000000000003e-05 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912598+0000", 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0046e-05 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912604+0000", 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9909999999999999e-06 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912650+0000", 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6013000000000001e-05 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912657+0000", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7429999999999996e-06 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145268+0000", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2326112730000001 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145425+0000", 2024-09-13T14:56:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156794 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145584+0000", 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158839 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150428+0000", 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048441869999999998 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172958+0000", 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022530171000000002 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178611+0000", 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056524990000000001 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208220+0000", 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029609139999999999 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208377+0000", 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015774100000000001 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209770+0000", 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001392475 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100028 9.10 9:0bded79a:::benchmark_data_smithi017_339564_object100027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.820763989, 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.300064551, 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100028 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912618+0000", 2024-09-13T14:56:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912662+0000", 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4375000000000001e-05 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912671+0000", 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5550000000000003e-06 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912676+0000", 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8749999999999999e-06 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912725+0000", 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9576000000000001e-05 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912733+0000", 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4150000000000002e-06 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148624+0000", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2358912049999999 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148772+0000", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147727 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148947+0000", 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175921 2024-09-13T14:56:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152859+0000", 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039114140000000002 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173020+0000", 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020160900999999998 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178657+0000", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005637268 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209824+0000", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.031167007 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.210229+0000", 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040503799999999998 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212682+0000", 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024528060000000001 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100040 9.10 9:0f8c354b:::benchmark_data_smithi017_339564_object100039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.82069625999998, 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.302457982, 2024-09-13T14:56:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100040 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3000000000000002e-07 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:56:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912719+0000", 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3429000000000001e-05 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912724+0000", 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9490000000000004e-06 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912727+0000", 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912758+0000", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0531000000000003e-05 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912768+0000", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0879e-05 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150460+0000", 2024-09-13T14:56:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2376911479999999 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150598+0000", 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013841999999999999 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150758+0000", 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000159479 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:56:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152866+0000", 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021080199999999999 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173065+0000", 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020199238000000001 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178693+0000", 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056281609999999996 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212776+0000", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034082685000000001 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212896+0000", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120565 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.215143+0000", 2024-09-13T14:56:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022468229999999998 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:56:13.126 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:56:13.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.127 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:12 smithi189 ceph-mon[90992]: pgmap v1529: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:56:13.127 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:13.127 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:13.127 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0074 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 126301 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 87.7018 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 40.7849 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 231.188 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 27.5 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1403 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 652.558 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3699 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 440 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.011398 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0129755 2024-09-13T14:56:13.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.474113 2024-09-13T14:56:13.129 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00143583 2024-09-13T14:56:13.129 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:56:13.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:13.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:13.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:13.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:13.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:13.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:13.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:13.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:13.872 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:13.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:13.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:14.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:13 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1761448706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:14.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:14.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:13 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1761448706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:13 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1761448706' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:14.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:14.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:14.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:14.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:14.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:14.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:14.946 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.982 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:14.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:14 smithi189 ceph-mon[90992]: pgmap v1530: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 943 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T14:56:15.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:15.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:14 smithi017 ceph-mon[103001]: pgmap v1530: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 943 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T14:56:15.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:14 smithi017 ceph-mon[107556]: pgmap v1530: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 943 op/s; 30 KiB/s, 0 objects/s recovering 2024-09-13T14:56:15.219 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.248 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:15.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:15.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:15.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:15.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:15.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:15.639 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.666 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:15.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:56:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:15.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:15.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:15.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:15.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:16.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.103 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:16.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:16.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:16.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:16.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:16.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:16.408 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:56:16.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:16.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:16.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:16.694 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:16.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:16.818 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:56:16.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:16.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.844 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:16.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:16.945 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:16.961 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:16 smithi189 ceph-mon[90992]: pgmap v1531: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 58 MiB/s wr, 942 op/s 2024-09-13T14:56:16.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:17.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:17.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:17.138 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:16 smithi017 ceph-mon[103001]: pgmap v1531: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 58 MiB/s wr, 942 op/s 2024-09-13T14:56:17.138 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:16 smithi017 ceph-mon[107556]: pgmap v1531: 109 pgs: 109 active+clean; 7.7 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 58 MiB/s wr, 942 op/s 2024-09-13T14:56:17.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:17.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:17.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:17] "GET /metrics HTTP/1.1" 200 36728 "" "Prometheus/2.51.0" 2024-09-13T14:56:17.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.565 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:17.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:17.820 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:17.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:17.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:17.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:18.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:18.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:18.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:18.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:18.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:18.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:18.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:18.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:18.658 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:18.701 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:18.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:56:18.800 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:18 smithi189 ceph-mon[90992]: pgmap v1532: 109 pgs: 109 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 53 MiB/s wr, 948 op/s 2024-09-13T14:56:18.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:18.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:18.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:18.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:18 smithi017 ceph-mon[103001]: pgmap v1532: 109 pgs: 109 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 53 MiB/s wr, 948 op/s 2024-09-13T14:56:18.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:18 smithi017 ceph-mon[107556]: pgmap v1532: 109 pgs: 109 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 53 MiB/s wr, 948 op/s 2024-09-13T14:56:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:18.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:19.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.116 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.218 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:19.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:19.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:19.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:19.466 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:19.466 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:19.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:19.528 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:19.629 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.652 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:19.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:19.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:19.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:19.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:19.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:19.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:19.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:20.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:20.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:20.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:56:20.169 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:20.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.349 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:20.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.391 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:20.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:20.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:20.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:20.577 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:20.627 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.663 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.765 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.797 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:20.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:20 smithi017 ceph-mon[103001]: pgmap v1533: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 943 op/s 2024-09-13T14:56:20.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:20 smithi017 ceph-mon[107556]: pgmap v1533: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 943 op/s 2024-09-13T14:56:21.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:20 smithi189 ceph-mon[90992]: pgmap v1533: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 943 op/s 2024-09-13T14:56:21.031 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:21.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:21.168 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:21.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:21.302 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.338 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:21.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:21.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:21.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:21.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:21.735 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:56:21.739 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:21.767 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:21.869 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:21.900 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:21.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3512453591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:21.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:21.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3512453591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:21.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:22.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3512453591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:22.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:56:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:22.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:22.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:22.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:22.286 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:22.322 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:22.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:22.460 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:22.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:22.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:22.702 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:22.746 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:22.834 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:22.834 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:22.847 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:22.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[103001]: pgmap v1534: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 86 B/s rd, 25 MiB/s wr, 822 op/s 2024-09-13T14:56:22.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:56:22.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[103001]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:56:22.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[107556]: pgmap v1534: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 86 B/s rd, 25 MiB/s wr, 822 op/s 2024-09-13T14:56:22.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:56:22.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:22 smithi017 ceph-mon[107556]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:56:22.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:22.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:22.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:22 smithi189 ceph-mon[90992]: pgmap v1534: 109 pgs: 109 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 86 B/s rd, 25 MiB/s wr, 822 op/s 2024-09-13T14:56:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:56:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:22 smithi189 ceph-mon[90992]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:56:23.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:23.123 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:23.153 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:23.287 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.389 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:23.434 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:23.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:23.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:23.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:23.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:23.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:23.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:24.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:24.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:24.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:24.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:56:24.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:24.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.376 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.410 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:24.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:24.511 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:24.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:24.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:24.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:24.901 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:24.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.938 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:24.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:24 smithi017 ceph-mon[103001]: pgmap v1536: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:24 smithi017 ceph-mon[107556]: pgmap v1536: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:24 smithi189 ceph-mon[90992]: pgmap v1536: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:25.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:25.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:25.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.215 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:25.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:25.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:25.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:25.452 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:25.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:25.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:25.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:25.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:25.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:25.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.892 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:25.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:26.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:26.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:26.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:26.271 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.307 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:26.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:26.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:26.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:26.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:26.687 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:26.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:26.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:26.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:26.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:27.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:26 smithi189 ceph-mon[90992]: pgmap v1537: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:27.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:27.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:26 smithi017 ceph-mon[103001]: pgmap v1537: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:26 smithi017 ceph-mon[107556]: pgmap v1537: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 1.9 MiB/s wr, 974 op/s 2024-09-13T14:56:27.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:27.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:27.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.510 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:27] "GET /metrics HTTP/1.1" 200 36725 "" "Prometheus/2.51.0" 2024-09-13T14:56:27.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:27.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:27.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:27.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:27.843 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.941 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:27.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:28.014 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:56:28.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:28.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:28.055 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:56:28.056 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:28.091 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:56:28.091 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:28.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:28.132 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:56:28.134 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:56:28.134 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:28.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:28.174 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:56:28.174 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:28.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:56:28.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:28.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:56:28.241 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:28.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:28.280 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:56:28.280 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:28.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:28.412 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:28.427 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.427 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: pgmap v1538: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 371 KiB/s wr, 971 op/s 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2360130347' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.428 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1083417825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: pgmap v1538: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 371 KiB/s wr, 971 op/s 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2360130347' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1083417825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: pgmap v1538: 109 pgs: 109 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 371 KiB/s wr, 971 op/s 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2360130347' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1083417825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:28.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:28.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:56:28.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:28.591 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:28.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:28.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:28.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:28.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:28.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:28.952 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:28.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:29.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:29.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:29.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:29.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[103001]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:56:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:29 smithi017 ceph-mon[107556]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:56:29.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:29.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:29.414 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:29 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:29.414 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:29 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:56:29.414 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:29 smithi189 ceph-mon[90992]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:56:29.457 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.487 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.620 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.721 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.755 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:29.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:30.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:30.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:30.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:30.175 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:30.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:30 smithi017 ceph-mon[103001]: pgmap v1540: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:30.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:30 smithi017 ceph-mon[107556]: pgmap v1540: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:30.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:30 smithi189 ceph-mon[90992]: pgmap v1540: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:30.278 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.306 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:30.547 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:30.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:30.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.851 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:31.037 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95176 9.7 9:e2f344fa:::benchmark_data_smithi017_339564_object95175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 321.11505944499999, 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89153507300000001, 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:56:31.038 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95176 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.039 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917992+0000", 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918043+0000", 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1029999999999998e-05 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.040 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918064+0000", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1078e-05 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918073+0000", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6060000000000003e-06 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921343+0000", 2024-09-13T14:56:31.041 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003269597 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921356+0000", 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3698e-05 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809462+0000", 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88810571900000002 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.042 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809529+0000", 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7564999999999994e-05 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95188 9.6 9:6261a5c8:::benchmark_data_smithi017_339564_object95187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 321.11496177100003, 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.90364369099999997, 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.043 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95188 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.044 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6738e-05 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.045 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918136+0000", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8599999999999993e-06 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6580000000000001e-06 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943565+0000", 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025424162 2024-09-13T14:56:31.046 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943572+0000", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.985e-06 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808380+0000", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86480780800000001 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808541+0000", 2024-09-13T14:56:31.047 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000160958 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808712+0000", 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000170958 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.810740+0000", 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002028555 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.048 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.813879+0000", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031388950000000001 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821630+0000", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077507640000000003 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821692+0000", 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2106000000000005e-05 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.049 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821736+0000", 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3798999999999999e-05 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95192 9.16 9:6c6762a8:::benchmark_data_smithi017_339564_object95191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 321.11490655199998, 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.917525484, 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95192 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.051 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.052 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.052 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:56:31.052 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.052 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.052 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0748999999999999e-05 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918195+0000", 2024-09-13T14:56:31.053 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.153e-06 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918199+0000", 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.194e-06 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943779+0000", 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025580471 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.054 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943784+0000", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0289999999999996e-06 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826118+0000", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88233390700000003 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826264+0000", 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014587500000000001 2024-09-13T14:56:31.055 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826408+0000", 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014380600000000001 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828905+0000", 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002496631 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.056 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.830686+0000", 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00178167 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835583+0000", 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048965609999999998 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835635+0000", 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2522000000000003e-05 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.057 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835673+0000", 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7376999999999998e-05 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95195 9.16 9:6f91244f:::benchmark_data_smithi017_339564_object95194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 321.11484995799998, 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91756910000000003, 2024-09-13T14:56:31.058 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95195 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.059 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918203+0000", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918232+0000", 2024-09-13T14:56:31.060 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8778000000000001e-05 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918239+0000", 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8190000000000003e-06 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918243+0000", 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1069999999999998e-06 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.061 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943799+0000", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025555422000000001 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943802+0000", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.749e-06 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827614+0000", 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88381141699999999 2024-09-13T14:56:31.062 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827747+0000", 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013306200000000001 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827906+0000", 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159474 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828913+0000", 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001006369 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.831206+0000", 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022932680000000002 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835706+0000", 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045005210000000004 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.064 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835745+0000", 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8803000000000001e-05 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835773+0000", 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7801e-05 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.065 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179257 9.12 9:4ecfe978:::benchmark_data_smithi017_339564_object59784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.76880193100001, 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7357178289999999, 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179257 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.066 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264251+0000", 2024-09-13T14:56:31.067 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264257+0000", 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4439999999999996e-06 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264275+0000", 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8155000000000002e-05 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.068 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264288+0000", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2853e-05 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264350+0000", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1716000000000003e-05 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264366+0000", 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5863999999999999e-05 2024-09-13T14:56:31.069 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818721+0000", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.554355614 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818727+0000", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9699999999999996e-06 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822922+0000", 2024-09-13T14:56:31.070 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0041942770000001 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822928+0000", 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6710000000000002e-06 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935124+0000", 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.112195982 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.071 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935256+0000", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013175400000000001 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935638+0000", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000382023 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943650+0000", 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080123120000000006 2024-09-13T14:56:31.072 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967274+0000", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023624030000000001 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.998994+0000", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031719232999999999 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999066+0000", 2024-09-13T14:56:31.073 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2358000000000001e-05 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999970+0000", 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00090380899999999999 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179272 9.12 9:4bdebc4a:::benchmark_data_smithi017_339564_object59799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:56:31.074 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.73608355499999, 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7069958839999999, 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179272 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.075 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296969+0000", 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.076 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296974+0000", 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1889999999999998e-06 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296984+0000", 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0131e-05 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296990+0000", 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.981e-06 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.077 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297030+0000", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9907000000000003e-05 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297036+0000", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1199999999999999e-06 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818748+0000", 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.52171188899999998 2024-09-13T14:56:31.078 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818754+0000", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822950+0000", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004196181 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822956+0000", 2024-09-13T14:56:31.079 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6510000000000004e-06 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.937987+0000", 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.115030368 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938146+0000", 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159703 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.080 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938563+0000", 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041611300000000002 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946223+0000", 2024-09-13T14:56:31.081 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007660498 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967408+0000", 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021185239000000002 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003056+0000", 2024-09-13T14:56:31.082 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035647757000000002 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003120+0000", 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3798000000000001e-05 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003966+0000", 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00084632100000000003 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.083 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179274 9.12 9:4d2975ed:::benchmark_data_smithi017_339564_object59801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.72995284199999, 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7113344069999998, 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179274 2024-09-13T14:56:31.084 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.085 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303104+0000", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6600000000000001e-06 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303112+0000", 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5519999999999999e-06 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.086 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303117+0000", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7759999999999997e-06 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303166+0000", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9438000000000003e-05 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303173+0000", 2024-09-13T14:56:31.087 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3130000000000001e-06 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818774+0000", 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51560055900000001 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818779+0000", 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3959999999999997e-06 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.088 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822980+0000", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004200381 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822987+0000", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9269999999999999e-06 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.940936+0000", 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1179493199999999 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.089 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941074+0000", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013840599999999999 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941402+0000", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032734600000000001 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948293+0000", 2024-09-13T14:56:31.090 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068912189999999996 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967530+0000", 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019237275000000002 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013579+0000", 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046049113000000003 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.091 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013672+0000", 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2820999999999998e-05 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014435+0000", 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000763406 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179351 9.12 9:48a19431:::benchmark_data_smithi017_339564_object59878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.092 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.591115057, 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5735510970000002, 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179351 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.093 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441937+0000", 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.094 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441945+0000", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.763e-06 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441962+0000", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7119999999999999e-05 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441967+0000", 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.75e-06 2024-09-13T14:56:31.095 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442014+0000", 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6425999999999999e-05 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442021+0000", 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.126e-06 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.096 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818801+0000", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.37678042 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818807+0000", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8989999999999998e-06 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823009+0000", 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0042012419999999 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.097 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823015+0000", 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.477e-06 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943680+0000", 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1206647809999999 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.098 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943812+0000", 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132278 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.944158+0000", 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000345869 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951355+0000", 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071972260000000001 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.099 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967668+0000", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016312454000000001 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014476+0000", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046808529000000001 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014559+0000", 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2373999999999994e-05 2024-09-13T14:56:31.100 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015490+0000", 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00093137299999999999 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179385 9.12 9:4cb1548a:::benchmark_data_smithi017_339564_object59912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.53324812599999, 2024-09-13T14:56:31.101 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5163587010000001, 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179385 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.102 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499805+0000", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499809+0000", 2024-09-13T14:56:31.103 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5070000000000002e-06 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499825+0000", 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5942000000000001e-05 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499831+0000", 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6790000000000002e-06 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.104 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499889+0000", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8090000000000001e-05 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499897+0000", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2649999999999996e-06 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818835+0000", 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31893799299999998 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.105 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818873+0000", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8136000000000002e-05 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823068+0000", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004194765 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823075+0000", 2024-09-13T14:56:31.106 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946257+0000", 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1231814090000001 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946386+0000", 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012880199999999999 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.107 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946701+0000", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000315719 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954875+0000", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081736080000000006 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967813+0000", 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012937664 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.108 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015557+0000", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047744885000000001 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015636+0000", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8121000000000006e-05 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016164+0000", 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00052882700000000005 2024-09-13T14:56:31.109 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179396 9.12 9:4f84ec82:::benchmark_data_smithi017_339564_object59923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.50509902900001, 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4892925720000001, 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.110 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179396 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:56:31.111 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527953+0000", 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527959+0000", 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2009999999999996e-06 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.112 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527974+0000", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4151999999999999e-05 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527979+0000", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4029999999999998e-06 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528038+0000", 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8718000000000003e-05 2024-09-13T14:56:31.113 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528051+0000", 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2835e-05 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818899+0000", 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.29084879699999999 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.114 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818906+0000", 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2369999999999997e-06 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823104+0000", 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004197929 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823110+0000", 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9099999999999999e-06 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.115 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948322+0000", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1252110909999999 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948445+0000", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123237 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948787+0000", 2024-09-13T14:56:31.116 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034259500000000001 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954934+0000", 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061466480000000002 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967970+0000", 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013036443 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.117 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016200+0000", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048229669000000003 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016731+0000", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053046400000000002 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017247+0000", 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00051684400000000003 2024-09-13T14:56:31.118 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179450 9.12 9:494d5338:::benchmark_data_smithi017_339564_object59977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 219.22040702300001, 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2051270810000001, 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.119 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179450 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.120 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812644+0000", 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812653+0000", 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7709999999999995e-06 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.121 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812673+0000", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9281000000000001e-05 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812681+0000", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5040000000000002e-06 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812744+0000", 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2874999999999997e-05 2024-09-13T14:56:31.122 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812756+0000", 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2223e-05 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818929+0000", 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061728369999999996 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.123 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818935+0000", 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0499999999999997e-06 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823287+0000", 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004352226 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823294+0000", 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3570000000000003e-06 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.124 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951389+0000", 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.12809574 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.125 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951538+0000", 2024-09-13T14:56:31.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014834099999999999 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.952029+0000", 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00049101500000000003 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.955950+0000", 2024-09-13T14:56:31.127 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039217050000000002 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.968099+0000", 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012148681999999999 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017285+0000", 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049186344 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.128 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017360+0000", 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4738000000000002e-05 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017774+0000", 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041378699999999999 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.129 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236410 9.12 9:4f0cdf2c:::benchmark_data_smithi017_339564_object116937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.40873786, 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0220686269999999, 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236410 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.130 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624314+0000", 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:56:31.131 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624321+0000", 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8889999999999996e-06 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624332+0000", 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0993e-05 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.132 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624338+0000", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4650000000000002e-06 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624435+0000", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6966999999999995e-05 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624452+0000", 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7719000000000001e-05 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.133 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.617701+0000", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99324826399999999 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.617790+0000", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9486000000000003e-05 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.618219+0000", 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042904299999999999 2024-09-13T14:56:31.134 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625381+0000", 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071616340000000001 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646269+0000", 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020888259999999999 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.135 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646325+0000", 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6217000000000001e-05 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646360+0000", 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4588000000000002e-05 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646385+0000", 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4833000000000001e-05 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.136 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236419 9.12 9:48d9d9e2:::benchmark_data_smithi017_339564_object116946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.40527859100001, 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0186878989999999, 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236419 2024-09-13T14:56:31.137 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.138 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627778+0000", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7699999999999999e-06 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627787+0000", 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5580000000000001e-06 2024-09-13T14:56:31.139 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627791+0000", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1420000000000003e-06 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.629765+0000", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019737880000000002 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.629773+0000", 2024-09-13T14:56:31.140 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4489999999999999e-06 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620327+0000", 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99055368200000005 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620388+0000", 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1173999999999994e-05 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.141 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620620+0000", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000231506 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627047+0000", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064277859999999996 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646297+0000", 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019249596000000001 2024-09-13T14:56:31.142 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646417+0000", 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000119707 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646443+0000", 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6469999999999999e-05 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.143 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646463+0000", 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0013999999999998e-05 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236481 9.12 9:4a8c469b:::benchmark_data_smithi017_339564_object117008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.33910879499999, 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95271523700000005, 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.144 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236481 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:56:31.145 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693947+0000", 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7669999999999999e-06 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.146 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693968+0000", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0607000000000001e-05 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693972+0000", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.755e-06 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.694008+0000", 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5969000000000001e-05 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.147 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.694013+0000", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6550000000000001e-06 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.621921+0000", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92790775199999997 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.621992+0000", 2024-09-13T14:56:31.148 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0696000000000002e-05 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.622313+0000", 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032082699999999998 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627054+0000", 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047417249999999996 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.149 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646496+0000", 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019442073000000001 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646606+0000", 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000109664 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646630+0000", 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3507999999999999e-05 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646660+0000", 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0679000000000002e-05 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236525 9.12 9:49746eac:::benchmark_data_smithi017_339564_object117052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.29563829400001, 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.90935570600000004, 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.151 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236525 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737415+0000", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737422+0000", 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.498e-06 2024-09-13T14:56:31.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737433+0000", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0361e-05 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737438+0000", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1499999999999998e-06 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.744319+0000", 2024-09-13T14:56:31.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068809559999999997 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.744327+0000", 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7329999999999992e-06 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.623594+0000", 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87926612400000004 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.623681+0000", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.721e-05 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.624015+0000", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033437000000000001 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630579+0000", 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0065640789999999996 2024-09-13T14:56:31.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646523+0000", 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015943969999999998 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646698+0000", 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000174659 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.157 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646737+0000", 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9223e-05 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646771+0000", 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4217999999999997e-05 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236567 9.12 9:4f71d45d:::benchmark_data_smithi017_339564_object117094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.158 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.246447233, 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86031424400000001, 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236567 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:56:31.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786606+0000", 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786611+0000", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6069999999999998e-06 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786626+0000", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4457999999999999e-05 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786632+0000", 2024-09-13T14:56:31.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786707+0000", 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4763000000000002e-05 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786716+0000", 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2520000000000005e-06 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625443+0000", 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83872685999999996 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625523+0000", 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9893000000000002e-05 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625780+0000", 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025712100000000002 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.635877+0000", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010096905 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646804+0000", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010927004000000001 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646846+0000", 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2595000000000001e-05 2024-09-13T14:56:31.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646893+0000", 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6697999999999999e-05 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646921+0000", 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8022e-05 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.165 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236603 9.12 9:4a904420:::benchmark_data_smithi017_339564_object117130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.20380865999999, 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83652915900000002, 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236603 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829244+0000", 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829249+0000", 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8359999999999999e-06 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829258+0000", 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4889999999999995e-06 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829261+0000", 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.636e-06 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829299+0000", 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7959e-05 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829306+0000", 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9249999999999998e-06 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627087+0000", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.797780716 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627155+0000", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7937999999999999e-05 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627475+0000", 2024-09-13T14:56:31.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032002899999999999 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.635884+0000", 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084087950000000002 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646982+0000", 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011098498 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665713+0000", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018730268000000001 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665749+0000", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6667e-05 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665774+0000", 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5150000000000001e-05 2024-09-13T14:56:31.172 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236628 9.12 9:48d6fa8d:::benchmark_data_smithi017_339564_object117155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.17294902, 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80943090500000003, 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236628 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.176 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:56:31.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860104+0000", 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860110+0000", 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8539999999999999e-06 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860124+0000", 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3287e-05 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860129+0000", 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6590000000000004e-06 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860191+0000", 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1199999999999997e-05 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860200+0000", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.149e-06 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.628814+0000", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76861446499999997 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.628907+0000", 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3015000000000003e-05 2024-09-13T14:56:31.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.629214+0000", 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00030697 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639163+0000", 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0099489750000000005 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.180 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.647021+0000", 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078578490000000001 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669475+0000", 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022453923000000001 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669506+0000", 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1111000000000001e-05 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669536+0000", 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9697999999999999e-05 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236642 9.12 9:48b908b1:::benchmark_data_smithi017_339564_object117169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.16127812299999, 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79782054800000002, 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.182 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236642 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871775+0000", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871779+0000", 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7339999999999999e-06 2024-09-13T14:56:31.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871790+0000", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0104e-05 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871796+0000", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.392e-06 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.872081+0000", 2024-09-13T14:56:31.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00028549499999999998 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.872091+0000", 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3980000000000005e-06 2024-09-13T14:56:31.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630615+0000", 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75852406299999997 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630680+0000", 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5501999999999997e-05 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.631003+0000", 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032254099999999997 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639191+0000", 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081879980000000002 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.647059+0000", 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078676460000000007 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669556+0000", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022497663000000001 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669579+0000", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3068e-05 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669596+0000", 2024-09-13T14:56:31.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6988999999999999e-05 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236688 9.12 9:4be6dfa1:::benchmark_data_smithi017_339564_object117215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 155.104199983, 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74087705599999998, 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:56:31.190 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236688 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:56:31.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928853+0000", 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928858+0000", 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.826e-06 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:56:31.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928868+0000", 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0648e-05 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928875+0000", 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.263e-06 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928951+0000", 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6817000000000007e-05 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928962+0000", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0577999999999999e-05 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632359+0000", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70339721499999996 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632445+0000", 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6211999999999994e-05 2024-09-13T14:56:31.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632673+0000", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022758300000000001 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639196+0000", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006522506 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.653018+0000", 2024-09-13T14:56:31.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013822723 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669682+0000", 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016663846 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669705+0000", 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2875e-05 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:56:31.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669731+0000", 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5918000000000002e-05 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:56:31.197 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:56:31.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:31.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:31.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:31.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:31.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:31.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:31.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:31.527 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:56:31.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:31.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:31.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:31.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:31.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3237688846' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:31 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:31.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:31.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:31.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3237688846' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:31 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3237688846' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:31 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:56:31.929 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:32.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:32.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.182 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:32.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:56:32.284 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:32.312 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:32.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.455 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:32.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:32.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:56:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:32.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:32.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:32.784 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[103001]: pgmap v1541: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:32.784 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:56:32.784 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[103001]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:56:32.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[107556]: pgmap v1541: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:32.785 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:56:32.786 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:32 smithi017 ceph-mon[107556]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:56:32.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:32 smithi189 ceph-mon[90992]: pgmap v1541: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-13T14:56:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:32 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:56:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:32 smithi189 ceph-mon[90992]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:56:33.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.043 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.145 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.172 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:33.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.273 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:33.307 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:33.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:33.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:33.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:33.559 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:33.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:33.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:33.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.777 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:33.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:56:33.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:33.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:34.022 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:34.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:34.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:34.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:34.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.330 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.576 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.610 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:34 smithi017 ceph-mon[103001]: pgmap v1543: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:56:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:34 smithi017 ceph-mon[107556]: pgmap v1543: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:56:34.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:34.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:34.857 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.894 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.996 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:34 smithi189 ceph-mon[90992]: pgmap v1543: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:56:35.025 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:35.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:35.267 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:35.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:35.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:35.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:35.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:35.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:35.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.825 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:35.858 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:35.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.006 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:36.007 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:36.008 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:36.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.111 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.144 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:36.289 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:36.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:36.421 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:36.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:36.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:36.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:36 smithi017 ceph-mon[107556]: pgmap v1544: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:56:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1535579160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:36 smithi017 ceph-mon[103001]: pgmap v1544: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:56:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1535579160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.802 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:36.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:36.958 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:37.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:36 smithi189 ceph-mon[90992]: pgmap v1544: 109 pgs: 109 active+clean; 6.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:56:37.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1535579160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:37.060 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.088 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.189 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:37.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:37.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:37.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:37.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:37] "GET /metrics HTTP/1.1" 200 36725 "" "Prometheus/2.51.0" 2024-09-13T14:56:37.458 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:37.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:37.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:37.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:37.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:37.774 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:37.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:37.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:38.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:38.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:38.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:38.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:38.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:38.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:38.544 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:56:38.585 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:38.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:38.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:38.849 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[103001]: pgmap v1545: 109 pgs: 109 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:56:38.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/420005902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:38.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:38.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[107556]: pgmap v1545: 109 pgs: 109 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:56:38.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/420005902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:38.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:38 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:38.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:38 smithi189 ceph-mon[90992]: pgmap v1545: 109 pgs: 109 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:56:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/420005902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:38 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:56:39.013 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:39.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.147 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:39.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.519 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:56:39.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:39.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:39.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:39.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:39.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:39.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:56:39.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:39 smithi017 ceph-mon[103001]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:56:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:56:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:39 smithi017 ceph-mon[107556]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:56:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:39.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:56:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:39 smithi189 ceph-mon[90992]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:56:40.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.110 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:40.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:40.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:40.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:40.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:40.506 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.649 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:40.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.750 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.786 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:40 smithi017 ceph-mon[103001]: pgmap v1547: 109 pgs: 109 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:40 smithi017 ceph-mon[107556]: pgmap v1547: 109 pgs: 109 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:41.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:40 smithi189 ceph-mon[90992]: pgmap v1547: 109 pgs: 109 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-13T14:56:41.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.050 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:41.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.185 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:41.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:41.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:41.432 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:41.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.573 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:41.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:41.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:41.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:41.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:41.969 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.011 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.142 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:42.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:42.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.426 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:42.606 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:42.631 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:42.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.710 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.811 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.856 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[103001]: pgmap v1548: 109 pgs: 109 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:56:42.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:42.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:42.878 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[107556]: pgmap v1548: 109 pgs: 109 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:56:42.878 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:42.878 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:42.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.997 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:42.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:42 smithi189 ceph-mon[90992]: pgmap v1548: 109 pgs: 109 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-09-13T14:56:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:56:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:43.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:43.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:43.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:43.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:43.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:43.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.557 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.659 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:43.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:43.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:43.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:43.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:44.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:44.159 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:44.160 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:44.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:44.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:44.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:44.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:44.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:44.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.529 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:44.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:44.597 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:56:44.630 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:44.665 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:44.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:44.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[103001]: pgmap v1549: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1163001533' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[107556]: pgmap v1549: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1163001533' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.903 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:44.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:56:44.941 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:44 smithi189 ceph-mon[90992]: pgmap v1549: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:56:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:44 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1163001533' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:45.042 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:45.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:45.315 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.350 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:45.453 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:45.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:45.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:45.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[103001]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:56:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:45 smithi017 ceph-mon[107556]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:56:45.726 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:45.763 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:45.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:45.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:46.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:46.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:45 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:46.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:45 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:46.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:45 smithi189 ceph-mon[90992]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:56:46.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:46.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:46.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:46.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:46.424 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.461 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:46.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:46.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.594 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:46.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:46.727 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:46.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.866 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:46.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:46 smithi017 ceph-mon[103001]: pgmap v1551: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:56:46.868 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:46 smithi017 ceph-mon[107556]: pgmap v1551: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:56:46.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:46 smithi189 ceph-mon[90992]: pgmap v1551: 109 pgs: 109 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:56:47.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:47.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:47.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:47.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.544 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:47.609 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:47] "GET /metrics HTTP/1.1" 200 36721 "" "Prometheus/2.51.0" 2024-09-13T14:56:47.685 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.723 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:47.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:47.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.964 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:48.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:48.104 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:48.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:48.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:48.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:48.273 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:48.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:48.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:48.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:48.648 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:56:48.649 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:48.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:48.685 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:56:48.685 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:48.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:48.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:48.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:56:48.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:48.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:56:48.769 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:48.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:48.810 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:56:48.811 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.834 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:48.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:48.849 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:56:48.849 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:48.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:48 smithi017 ceph-mon[103001]: pgmap v1552: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:48.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:48 smithi017 ceph-mon[107556]: pgmap v1552: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:48.876 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:48.903 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:56:48.903 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:48.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:56:48.935 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:48.940 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:56:48.940 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:48.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:48 smithi189 ceph-mon[90992]: pgmap v1552: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:49.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.216 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.246 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:49.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:49.347 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.376 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:49.478 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:49.508 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:49.606 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:56:49.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:49.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:49.753 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:49.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/656186070' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:56:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/656186070' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:56:49.957 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:49.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:50.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/656186070' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:56:50.059 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:50.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:50.194 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.237 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:50.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:50.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:50.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:50.505 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:56:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:56:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:50.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[103001]: pgmap v1553: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[103001]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[107556]: pgmap v1553: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:56:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:50 smithi017 ceph-mon[107556]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:56:50.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.765 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:50.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.905 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:50 smithi189 ceph-mon[90992]: pgmap v1553: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:50 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4254133582' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:56:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:50 smithi189 ceph-mon[90992]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:56:51.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:51.045 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:51.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:51.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:51.313 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:51.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:51.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:51.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:51.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:51.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:51.700 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:51.739 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:51.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:51.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:51.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:51.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:52.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:56:52.064 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:52.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:52.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:52.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:52.314 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:52.416 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:52.449 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:52.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:52.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:52.592 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:52.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:52 smithi017 ceph-mon[103001]: pgmap v1555: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:56:52.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:52 smithi017 ceph-mon[107556]: pgmap v1555: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:56:52.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:52.838 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:52.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:52.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:52.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.001 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:53.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:52 smithi189 ceph-mon[90992]: pgmap v1555: 109 pgs: 109 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:56:53.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:53.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.240 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.271 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:53.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:53.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:53.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:53.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:53.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:53.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:53.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:53.830 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:53.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:53.969 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:54.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:54.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:54.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.379 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:54.402 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:56:54.402 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:56:54.402 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:56:54.402 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:56:54.402 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:56:54.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:54.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:54.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:54 smithi017 ceph-mon[103001]: pgmap v1556: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:54.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:54 smithi017 ceph-mon[107556]: pgmap v1556: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:54.713 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:54.748 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:54.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:54 smithi189 ceph-mon[90992]: pgmap v1556: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-13T14:56:55.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:55.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:55.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:55.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:55.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:55.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.439 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:55.540 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.579 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:55.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:55.849 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:55.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.985 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:55.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:56.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.114 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:56.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:56.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:56.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:56.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:56.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:56.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:56.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:56.633 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:56:56.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:56.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:56 smithi017 ceph-mon[103001]: pgmap v1557: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:56.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:56 smithi017 ceph-mon[107556]: pgmap v1557: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:56.791 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:56.827 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:56.929 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:56.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:56 smithi189 ceph-mon[90992]: pgmap v1557: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:56:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:57.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:57.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:57.202 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.238 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:57.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:57.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:56:57.339 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.374 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:57.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:57.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:57.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:57.535 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:57.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.666 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:56:57] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.51.0" 2024-09-13T14:56:57.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.791 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.893 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.931 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:57.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[103001]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:56:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[107556]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:56:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:57 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3571349267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:57 smithi189 ceph-mon[90992]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:56:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:58.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:58.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:58.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:58.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:58.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:58.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:58.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:58.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:58.605 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.638 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:58.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:58.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:58.780 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:58.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:56:58 smithi017 ceph-mon[103001]: pgmap v1559: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:56:58.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:56:58 smithi017 ceph-mon[107556]: pgmap v1559: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:56:58.881 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.917 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:58.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:56:58 smithi189 ceph-mon[90992]: pgmap v1559: 109 pgs: 109 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:56:59.018 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.051 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:59.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:59.152 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:59.187 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:59.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:59.321 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:59.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:59.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:59.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.717 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.750 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:56:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:56:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:00.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:00.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:00.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:00.324 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:00.361 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:00.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:00.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:00.576 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:00.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:00.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:57:00.692 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:57:00.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:00.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:00 smithi189 ceph-mon[90992]: pgmap v1560: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:57:00.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:57:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.930 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:00.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:00 smithi017 ceph-mon[103001]: pgmap v1560: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:57:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:57:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:00 smithi017 ceph-mon[107556]: pgmap v1560: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:57:00.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:57:01.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:01.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:01.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:01.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:01.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:01.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:01.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:01.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:01.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:01.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:01.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.886 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:01.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:01.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:57:01.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:01 smithi017 ceph-mon[103001]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:57:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:57:01.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:01 smithi017 ceph-mon[107556]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:57:01.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4189665423' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:57:02.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:01 smithi189 ceph-mon[90992]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:57:02.032 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:02.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:02.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:02.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.301 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:02.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:02.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:02.538 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:02.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.669 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:02.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:02.802 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.831 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.862 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:02 smithi017 ceph-mon[103001]: pgmap v1562: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-13T14:57:02.863 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:02 smithi017 ceph-mon[107556]: pgmap v1562: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-13T14:57:02.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:02.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:03.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:57:03.001 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:03.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:02 smithi189 ceph-mon[90992]: pgmap v1562: 109 pgs: 109 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-09-13T14:57:03.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:57:03.018 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:03.043 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:57:03.043 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:03.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.081 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:57:03.081 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:03.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:57:03.114 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:03.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:03.147 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:57:03.148 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:03.177 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:57:03.177 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:03.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:57:03.211 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:03.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.256 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:03.358 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.404 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:03.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:03.644 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:03.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.777 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:03.777 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:03.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:03.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:03.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3735558407' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3735558407' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:03.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3735558407' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:04.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:04.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:04.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:04.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:04.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:04.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:04.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:57:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.767 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:04.805 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:04.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:04 smithi017 ceph-mon[107556]: pgmap v1563: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:04.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:04 smithi017 ceph-mon[103001]: pgmap v1563: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:04.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:04 smithi189 ceph-mon[90992]: pgmap v1563: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:05.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:05.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:05.223 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:05.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:05.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:05.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.645 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:05.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:05.762 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:05.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:57:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:05.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:05.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:05.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:06.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.132 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:06.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:06.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:06.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.509 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.532 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.676 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:06.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:06.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:06.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:06.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:06.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:07.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:06 smithi189 ceph-mon[90992]: pgmap v1564: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:57:07.055 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.076 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:07.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:07.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:06 smithi017 ceph-mon[107556]: pgmap v1564: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:57:07.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:06 smithi017 ceph-mon[103001]: pgmap v1564: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:57:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:07.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:07.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.335 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.437 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.478 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:07.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:07.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:07.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:07.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:07] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.51.0" 2024-09-13T14:57:07.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:07.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:07.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:07.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:07.897 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:07.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:07.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:08.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:08.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:08.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:08.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:08.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.452 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:08.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:08.517 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:08.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:08.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:08.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:08.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:08.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:08 smithi017 ceph-mon[103001]: pgmap v1565: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:57:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:08 smithi017 ceph-mon[107556]: pgmap v1565: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:57:08.995 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:08 smithi189 ceph-mon[90992]: pgmap v1565: 109 pgs: 109 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:57:09.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.031 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:09.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.165 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.268 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.302 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:09.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.303 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:09.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:09.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:09.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:09.551 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.583 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:09.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:09.717 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:09.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:09.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:09.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:09.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:10.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:10.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.237 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.288 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:10.391 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.430 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:10.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:10.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:10.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:10.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:10.714 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:10.777 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:57:10.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:10.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:10.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.004 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:11.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[103001]: pgmap v1566: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3482738439' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[107556]: pgmap v1566: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3482738439' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.138 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:11.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:11.239 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:11.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:10 smithi189 ceph-mon[90992]: pgmap v1566: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:11.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3482738439' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.257 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:11.257 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 14:57:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a[78076]: logger=cleanup t=2024-09-13T14:57:10.827531582Z level=info msg="Completed cleanup jobs" duration=23.873454ms 2024-09-13T14:57:11.270 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:11.372 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.405 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:11.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:11.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:11.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.756 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:11.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:11.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:11.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:11.963 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:11.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:11 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:11 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:57:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:12.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:11 smithi189 ceph-mon[90992]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[103001]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:12.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:11 smithi017 ceph-mon[107556]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:57:12.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:12.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:12.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:12.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:12.346 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.379 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:12.481 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:12.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:12.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:12.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:12.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:12.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[103001]: pgmap v1568: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:12.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:12.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:12.831 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[107556]: pgmap v1568: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:12.831 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:12.831 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:12.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:12.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:12 smithi189 ceph-mon[90992]: pgmap v1568: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-09-13T14:57:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:13.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:13.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:13.190 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:13.231 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:13.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:13.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:13.471 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.504 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:13.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:13.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:13.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:13.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:13.796 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:57:13.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:13.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:14.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:14.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:14.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:13 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2782778587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:13 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:13 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2782778587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:13 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.209 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:14.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:13 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2782778587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:13 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:57:14.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:14.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:14.448 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.483 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:14.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:14.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:57:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:14.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:14.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:14.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:14.865 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.902 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[107556]: pgmap v1569: 109 pgs: 109 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-13T14:57:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[107556]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:57:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[103001]: pgmap v1569: 109 pgs: 109 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-13T14:57:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:14 smithi017 ceph-mon[103001]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:57:15.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:15.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:15.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:57:15.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:15.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:14 smithi189 ceph-mon[90992]: pgmap v1569: 109 pgs: 109 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-13T14:57:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:14 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:57:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:14 smithi189 ceph-mon[90992]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:57:15.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:15.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:15.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:15.574 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:15.618 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:15.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:15.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.757 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:15.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:15.858 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:15.997 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.030 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:16.130 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:16.132 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.158 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:16.176 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:16.176 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:16.259 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:16.299 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:16.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:16.437 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:16.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:16.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:16.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:16.821 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.851 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:16.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:16 smithi017 ceph-mon[103001]: pgmap v1571: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:57:16.887 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:16 smithi017 ceph-mon[107556]: pgmap v1571: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:57:16.952 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.987 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:16.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:17.088 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.123 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:17.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:57:17.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:17.252 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:17.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:16 smithi189 ceph-mon[90992]: pgmap v1571: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-13T14:57:17.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:17.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:17.281 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:57:17.282 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:17.313 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:57:17.313 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:17.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:57:17.347 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:17.364 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:17.378 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:57:17.378 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:17.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:17.404 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:57:17.405 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:17.405 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:17] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.51.0" 2024-09-13T14:57:17.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:57:17.435 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:17.473 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:57:17.473 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:17.504 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.534 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:17.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:17.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:17.782 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:17.814 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:17.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:17.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:18.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:18.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:18.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:17 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3571852945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:17 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3571852945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:18.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:18.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:17 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3571852945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:18.328 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:18.359 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:18.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:18.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:18.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:18.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:18.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:18.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:18.771 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:18.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:18.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:18.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:19.033 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:19.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:18 smithi017 ceph-mon[103001]: pgmap v1572: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:57:19.061 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:18 smithi017 ceph-mon[107556]: pgmap v1572: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:57:19.063 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.165 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:19.201 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:18 smithi189 ceph-mon[90992]: pgmap v1572: 109 pgs: 109 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:57:19.304 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:19.339 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:19.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:19.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:57:19.580 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:19.611 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.712 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.742 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:19.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:19.881 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:20.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:20.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:20.118 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:20.150 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:20.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:20.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:20.403 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:20.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.581 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:20.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:20.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:20.830 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.863 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:20.965 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:21.100 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:21.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:20 smithi017 ceph-mon[103001]: pgmap v1573: 109 pgs: 109 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:57:21.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:20 smithi017 ceph-mon[107556]: pgmap v1573: 109 pgs: 109 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:57:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.129 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:21.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:21.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:20 smithi189 ceph-mon[90992]: pgmap v1573: 109 pgs: 109 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-13T14:57:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:21.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:21.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:21.513 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:21.547 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:21.649 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:21.679 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:21.767 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:21.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:21.840 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:57:21.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:21.957 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:22.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:21 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2772087543' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:21 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.091 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:21 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2772087543' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:21 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.129 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.231 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:21 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2772087543' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:21 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:22.269 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:22.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:22.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:22.510 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.541 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:22.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.643 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:22.677 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:22.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:22.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:22.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:22.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:23.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[103001]: pgmap v1574: 109 pgs: 109 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:23.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:23.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:23.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[103001]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:57:23.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[107556]: pgmap v1574: 109 pgs: 109 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:23.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:23.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:23.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:22 smithi017 ceph-mon[107556]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:57:23.092 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:23.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:22 smithi189 ceph-mon[90992]: pgmap v1574: 109 pgs: 109 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:22 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:23.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:22 smithi189 ceph-mon[90992]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:57:23.334 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.382 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:23.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.395 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:23.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:23.513 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:23.616 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.654 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:23.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.755 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:23.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:23.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:24.029 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.062 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:24.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:24.163 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.200 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:24.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:24.341 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:24.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.474 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:24.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:24.725 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:24.760 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:24.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:24.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:24.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:25.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:57:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:25.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:25.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:24 smithi017 ceph-mon[103001]: pgmap v1576: 109 pgs: 109 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:25.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:24 smithi017 ceph-mon[107556]: pgmap v1576: 109 pgs: 109 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:25.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:25.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:24 smithi189 ceph-mon[90992]: pgmap v1576: 109 pgs: 109 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:25.283 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:25.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:25.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:25.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:25.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.571 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:57:25.572 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:57:25.572 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:57:25.572 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:57:25.572 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:57:25.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:25.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:25.851 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.884 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:25.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.985 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:26.018 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:26.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:26.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:26.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.415 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:26.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:26.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:26.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:26.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:26.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:57:26.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:26.854 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:57:26.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:26.985 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[103001]: pgmap v1577: 109 pgs: 109 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:26.985 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3263676473' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:26.985 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:26.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[107556]: pgmap v1577: 109 pgs: 109 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:26.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3263676473' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:26.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:26.999 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.034 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:27.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.135 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:27.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:27.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:26 smithi189 ceph-mon[90992]: pgmap v1577: 109 pgs: 109 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:26 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3263676473' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:27.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:57:27.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:27.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:27.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:27.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:27.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:27.617 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:27] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-09-13T14:57:27.696 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:27.738 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:27.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.871 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:57:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[103001]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:57:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:57:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[107556]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:57:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:27.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:28.015 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:28.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.147 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:57:28.147 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:28.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:57:28.183 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:28.195 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:28.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:57:28.247 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:27 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:57:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:27 smithi189 ceph-mon[90992]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:57:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:28.287 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:57:28.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:28.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:28.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.315 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:57:28.315 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:28.344 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:57:28.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:28.378 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:57:28.379 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:28.396 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:28.411 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:57:28.411 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:28.427 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:28.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:28.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:28.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:28.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.799 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:57:28.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:57:28.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:29.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:29.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:29.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[103001]: pgmap v1579: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-09-13T14:57:29.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3631094263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:29.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:29.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[107556]: pgmap v1579: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-09-13T14:57:29.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3631094263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:29.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:29.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:28 smithi189 ceph-mon[90992]: pgmap v1579: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-09-13T14:57:29.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3631094263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:29.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:29.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:29.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:29.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:29.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:29.605 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:29.666 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:29.707 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.752 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:29.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:29.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:30.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.039 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:30.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:30.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:30.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:30.279 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.327 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:30.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:30.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:30.550 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:30.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:30.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:30.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:30.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:30.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:30 smithi189 ceph-mon[90992]: pgmap v1580: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:31.014 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:31.049 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:31.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:31.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:31.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:30 smithi017 ceph-mon[103001]: pgmap v1580: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:30 smithi017 ceph-mon[107556]: pgmap v1580: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-13T14:57:31.293 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.331 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:31.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.480 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:31.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:31.581 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:31.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:31.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:31.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:31.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:31.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.041 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.080 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:32.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:32.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:32.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:32.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:32.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.637 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:32.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:32.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:32.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:32.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:32.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:33.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:33.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:33.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:33.123 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:32 smithi189 ceph-mon[90992]: pgmap v1581: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:33.168 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:32 smithi017 ceph-mon[103001]: pgmap v1581: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:32 smithi017 ceph-mon[107556]: pgmap v1581: 109 pgs: 109 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:57:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.210 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:33.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:33.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:33.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:33.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:33.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:33.584 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.626 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:33.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:33.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:34.005 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:34.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:34.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:34.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:34.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:34.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:34.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:34.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:34.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.829 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.883 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:34.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:34.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:35.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:34 smithi017 ceph-mon[103001]: pgmap v1582: 109 pgs: 109 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:57:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:34 smithi017 ceph-mon[107556]: pgmap v1582: 109 pgs: 109 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:57:35.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:34 smithi189 ceph-mon[90992]: pgmap v1582: 109 pgs: 109 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-09-13T14:57:35.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:35.395 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:35.395 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:35.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:35.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:35.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:35.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:35.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:35.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:35.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[103001]: pgmap v1583: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:35 smithi017 ceph-mon[107556]: pgmap v1583: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:36.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:35 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:36.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:35 smithi189 ceph-mon[90992]: pgmap v1583: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-13T14:57:36.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.386 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.418 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:36.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:36.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:57:36.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:36.831 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:36.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:36.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:37.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.230 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.259 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:37.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:37.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.483 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:37.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:37] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.51.0" 2024-09-13T14:57:37.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:37.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:37.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:37.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.828 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.860 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:37.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.963 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.998 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.099 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:38.173 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:38.236 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.272 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.403 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:38.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:38.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:38.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:38.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.688 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:38.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:38.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:38.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:38 smithi017 ceph-mon[103001]: pgmap v1584: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-13T14:57:38.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:38 smithi017 ceph-mon[107556]: pgmap v1584: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-13T14:57:38.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:38.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:38 smithi189 ceph-mon[90992]: pgmap v1584: 109 pgs: 109 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-13T14:57:39.076 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.112 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:39.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.214 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.250 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:39.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.351 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:39.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:39.544 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:57:39.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:39.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:39.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:39.795 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:57:39.795 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1972033999' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:39.795 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:39 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:39.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1972033999' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:39.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:39 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:39.800 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:57:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.810 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:39.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:39.911 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:39.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1972033999' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:40.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:39 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:40.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:40.470 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:40.500 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:40.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:40.663 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1586,"stamp":"2024-09-13T14:57:39.540984+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2821461989,"num_objects":43008,"num_object_clones":0,"num_object_copies":129024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43008,"num_whiteouts":0,"num_read":5969,"num_read_kb":12982,"num_write":210531,"num_write_kb":8095888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":103,"num_bytes_recovered":8721938,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119822,"ondisk_log_size":119822,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15641072,"kb_used_data":8426032,"kb_used_omap":44,"kb_used_meta":7214739,"kb_avail":734320144,"statfs":{"total":767960285184,"available":751943827456,"internally_reserved":0,"allocated":8628256768,"data_stored":8608138735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7387892983},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-640090112,"num_objects":-9767,"num_object_clones":0,"num_object_copies":-29301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9767,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.072084"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":295,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.866931+0000","last_change":"2024-09-13T14:56:00.831620+0000","last_active":"2024-09-13T14:57:26.866931+0000","last_peered":"2024-09-13T14:57:26.866931+0000","last_clean":"2024-09-13T14:57:26.866931+0000","last_became_active":"2024-09-13T14:56:00.676063+0000","last_became_peered":"2024-09-13T14:56:00.676063+0000","last_unstale":"2024-09-13T14:57:26.866931+0000","last_undegraded":"2024-09-13T14:57:26.866931+0000","last_fullsized":"2024-09-13T14:57:26.866931+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.831575+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.831575+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.735233+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":849,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.852935+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:57:26.852935+0000","last_peered":"2024-09-13T14:57:26.852935+0000","last_clean":"2024-09-13T14:57:26.852935+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:57:26.852935+0000","last_undegraded":"2024-09-13T14:57:26.852935+0000","last_fullsized":"2024-09-13T14:57:26.852935+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":478,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830802+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:57:21.830802+0000","last_peered":"2024-09-13T14:57:21.830802+0000","last_clean":"2024-09-13T14:57:21.830802+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:57:21.830802+0000","last_undegraded":"2024-09-13T14:57:21.830802+0000","last_fullsized":"2024-09-13T14:57:21.830802+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":33,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824002+0000","last_change":"2024-09-13T14:56:04.574886+0000","last_active":"2024-09-13T14:57:21.824002+0000","last_peered":"2024-09-13T14:57:21.824002+0000","last_clean":"2024-09-13T14:57:21.824002+0000","last_became_active":"2024-09-13T14:56:00.674692+0000","last_became_peered":"2024-09-13T14:56:00.674692+0000","last_unstale":"2024-09-13T14:57:21.824002+0000","last_undegraded":"2024-09-13T14:57:21.824002+0000","last_fullsized":"2024-09-13T14:57:21.824002+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.574787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.574787+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:30.673194+0000","scrub_duration":0.0020770139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":407,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824890+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:57:21.824890+0000","last_peered":"2024-09-13T14:57:21.824890+0000","last_clean":"2024-09-13T14:57:21.824890+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:57:21.824890+0000","last_undegraded":"2024-09-13T14:57:21.824890+0000","last_fullsized":"2024-09-13T14:57:21.824890+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.897149+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:57:26.897149+0000","last_peered":"2024-09-13T14:57:26.897149+0000","last_clean":"2024-09-13T14:57:26.897149+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:57:26.897149+0000","last_undegraded":"2024-09-13T14:57:26.897149+0000","last_fullsized":"2024-09-13T14:57:26.897149+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":685,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842417+0000","last_change":"2024-09-13T14:56:01.351509+0000","last_active":"2024-09-13T14:57:26.842417+0000","last_peered":"2024-09-13T14:57:26.842417+0000","last_clean":"2024-09-13T14:57:26.842417+0000","last_became_active":"2024-09-13T14:56:00.675249+0000","last_became_peered":"2024-09-13T14:56:00.675249+0000","last_unstale":"2024-09-13T14:57:26.842417+0000","last_undegraded":"2024-09-13T14:57:26.842417+0000","last_fullsized":"2024-09-13T14:57:26.842417+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.351298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.351298+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:23.442259+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":135,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.884066+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:57:26.884066+0000","last_peered":"2024-09-13T14:57:26.884066+0000","last_clean":"2024-09-13T14:57:26.884066+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:57:26.884066+0000","last_undegraded":"2024-09-13T14:57:26.884066+0000","last_fullsized":"2024-09-13T14:57:26.884066+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":447,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.832706+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:57:21.832706+0000","last_peered":"2024-09-13T14:57:21.832706+0000","last_clean":"2024-09-13T14:57:21.832706+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:57:21.832706+0000","last_undegraded":"2024-09-13T14:57:21.832706+0000","last_fullsized":"2024-09-13T14:57:21.832706+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":33,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824002+0000","last_change":"2024-09-13T14:56:01.635272+0000","last_active":"2024-09-13T14:57:21.824002+0000","last_peered":"2024-09-13T14:57:21.824002+0000","last_clean":"2024-09-13T14:57:21.824002+0000","last_became_active":"2024-09-13T14:56:00.667926+0000","last_became_peered":"2024-09-13T14:56:00.667926+0000","last_unstale":"2024-09-13T14:57:21.824002+0000","last_undegraded":"2024-09-13T14:57:21.824002+0000","last_fullsized":"2024-09-13T14:57:21.824002+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.635189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.635189+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:28.334884+0000","scrub_duration":0.010355512000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":848,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.857349+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:57:26.857349+0000","last_peered":"2024-09-13T14:57:26.857349+0000","last_clean":"2024-09-13T14:57:26.857349+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:57:26.857349+0000","last_undegraded":"2024-09-13T14:57:26.857349+0000","last_fullsized":"2024-09-13T14:57:26.857349+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":509,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830005+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:57:21.830005+0000","last_peered":"2024-09-13T14:57:21.830005+0000","last_clean":"2024-09-13T14:57:21.830005+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:57:21.830005+0000","last_undegraded":"2024-09-13T14:57:21.830005+0000","last_fullsized":"2024-09-13T14:57:21.830005+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":33,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831010+0000","last_change":"2024-09-13T14:56:00.915923+0000","last_active":"2024-09-13T14:57:21.831010+0000","last_peered":"2024-09-13T14:57:21.831010+0000","last_clean":"2024-09-13T14:57:21.831010+0000","last_became_active":"2024-09-13T14:56:00.671638+0000","last_became_peered":"2024-09-13T14:56:00.671638+0000","last_unstale":"2024-09-13T14:57:21.831010+0000","last_undegraded":"2024-09-13T14:57:21.831010+0000","last_fullsized":"2024-09-13T14:57:21.831010+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.915835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.915835+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:52.043909+0000","scrub_duration":0.010872074000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"480'13021","reported_seq":13060,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.463931+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:57:38.463931+0000","last_peered":"2024-09-13T14:57:38.463931+0000","last_clean":"2024-09-13T14:57:38.463931+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:57:38.463931+0000","last_undegraded":"2024-09-13T14:57:38.463931+0000","last_fullsized":"2024-09-13T14:57:38.463931+0000","mapping_epoch":449,"log_start":"459'4400","ondisk_log_start":"459'4400","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8621,"log_dups_size":0,"ondisk_log_size":8621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172163072,"num_objects":2627,"num_object_clones":0,"num_object_copies":7881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13021,"num_write_kb":500736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":501,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831607+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:57:21.831607+0000","last_peered":"2024-09-13T14:57:21.831607+0000","last_clean":"2024-09-13T14:57:21.831607+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:57:21.831607+0000","last_undegraded":"2024-09-13T14:57:21.831607+0000","last_fullsized":"2024-09-13T14:57:21.831607+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":392,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824082+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:57:21.824082+0000","last_peered":"2024-09-13T14:57:21.824082+0000","last_clean":"2024-09-13T14:57:21.824082+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:57:21.824082+0000","last_undegraded":"2024-09-13T14:57:21.824082+0000","last_fullsized":"2024-09-13T14:57:21.824082+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":718,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.852223+0000","last_change":"2024-09-13T14:56:00.697476+0000","last_active":"2024-09-13T14:57:26.852223+0000","last_peered":"2024-09-13T14:57:26.852223+0000","last_clean":"2024-09-13T14:57:26.852223+0000","last_became_active":"2024-09-13T14:56:00.680491+0000","last_became_peered":"2024-09-13T14:56:00.680491+0000","last_unstale":"2024-09-13T14:57:26.852223+0000","last_undegraded":"2024-09-13T14:57:26.852223+0000","last_fullsized":"2024-09-13T14:57:26.852223+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:56.869434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:56.869434+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:32.392000+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"480'13307","reported_seq":13346,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.970193+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:57:37.970193+0000","last_peered":"2024-09-13T14:57:37.970193+0000","last_clean":"2024-09-13T14:57:37.970193+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:57:37.970193+0000","last_undegraded":"2024-09-13T14:57:37.970193+0000","last_fullsized":"2024-09-13T14:57:37.970193+0000","mapping_epoch":449,"log_start":"461'6276","ondisk_log_start":"461'6276","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13307,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":491,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831493+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:57:21.831493+0000","last_peered":"2024-09-13T14:57:21.831493+0000","last_clean":"2024-09-13T14:57:21.831493+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:57:21.831493+0000","last_undegraded":"2024-09-13T14:57:21.831493+0000","last_fullsized":"2024-09-13T14:57:21.831493+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.825850+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:57:21.825850+0000","last_peered":"2024-09-13T14:57:21.825850+0000","last_clean":"2024-09-13T14:57:21.825850+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:57:21.825850+0000","last_undegraded":"2024-09-13T14:57:21.825850+0000","last_fullsized":"2024-09-13T14:57:21.825850+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":508,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830809+0000","last_change":"2024-09-13T14:56:02.698903+0000","last_active":"2024-09-13T14:57:21.830809+0000","last_peered":"2024-09-13T14:57:21.830809+0000","last_clean":"2024-09-13T14:57:21.830809+0000","last_became_active":"2024-09-13T14:56:00.691242+0000","last_became_peered":"2024-09-13T14:56:00.691242+0000","last_unstale":"2024-09-13T14:57:21.830809+0000","last_undegraded":"2024-09-13T14:57:21.830809+0000","last_fullsized":"2024-09-13T14:57:21.830809+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.698817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.698817+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:11.424244+0000","scrub_duration":0.0077410969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"480'13175","reported_seq":13249,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.567749+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:57:37.567749+0000","last_peered":"2024-09-13T14:57:37.567749+0000","last_clean":"2024-09-13T14:57:37.567749+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:57:37.567749+0000","last_undegraded":"2024-09-13T14:57:37.567749+0000","last_fullsized":"2024-09-13T14:57:37.567749+0000","mapping_epoch":449,"log_start":"460'5956","ondisk_log_start":"460'5956","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7219,"log_dups_size":0,"ondisk_log_size":7219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177930240,"num_objects":2715,"num_object_clones":0,"num_object_copies":8145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13175,"num_write_kb":508480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":445,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831271+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:57:21.831271+0000","last_peered":"2024-09-13T14:57:21.831271+0000","last_clean":"2024-09-13T14:57:21.831271+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:57:21.831271+0000","last_undegraded":"2024-09-13T14:57:21.831271+0000","last_fullsized":"2024-09-13T14:57:21.831271+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":460,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830208+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:57:21.830208+0000","last_peered":"2024-09-13T14:57:21.830208+0000","last_clean":"2024-09-13T14:57:21.830208+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:57:21.830208+0000","last_undegraded":"2024-09-13T14:57:21.830208+0000","last_fullsized":"2024-09-13T14:57:21.830208+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":33,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.825741+0000","last_change":"2024-09-13T14:56:05.805624+0000","last_active":"2024-09-13T14:57:21.825741+0000","last_peered":"2024-09-13T14:57:21.825741+0000","last_clean":"2024-09-13T14:57:21.825741+0000","last_became_active":"2024-09-13T14:56:00.676524+0000","last_became_peered":"2024-09-13T14:56:00.676524+0000","last_unstale":"2024-09-13T14:57:21.825741+0000","last_undegraded":"2024-09-13T14:57:21.825741+0000","last_fullsized":"2024-09-13T14:57:21.825741+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:05.805551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:05.805551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:13.686450+0000","scrub_duration":0.028580083999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.5","version":"480'12911","reported_seq":12983,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.596070+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:57:37.596070+0000","last_peered":"2024-09-13T14:57:37.596070+0000","last_clean":"2024-09-13T14:57:37.596070+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:57:37.596070+0000","last_undegraded":"2024-09-13T14:57:37.596070+0000","last_fullsized":"2024-09-13T14:57:37.596070+0000","mapping_epoch":449,"log_start":"460'5733","ondisk_log_start":"460'5733","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172818432,"num_objects":2637,"num_object_clones":0,"num_object_copies":7911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12911,"num_write_kb":497536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":408,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824606+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:57:21.824606+0000","last_peered":"2024-09-13T14:57:21.824606+0000","last_clean":"2024-09-13T14:57:21.824606+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:57:21.824606+0000","last_undegraded":"2024-09-13T14:57:21.824606+0000","last_fullsized":"2024-09-13T14:57:21.824606+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":135,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.869644+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:57:26.869644+0000","last_peered":"2024-09-13T14:57:26.869644+0000","last_clean":"2024-09-13T14:57:26.869644+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:57:26.869644+0000","last_undegraded":"2024-09-13T14:57:26.869644+0000","last_fullsized":"2024-09-13T14:57:26.869644+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"480'13152","reported_seq":13224,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:35.910479+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:57:35.910479+0000","last_peered":"2024-09-13T14:57:35.910479+0000","last_clean":"2024-09-13T14:57:35.910479+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:57:35.910479+0000","last_undegraded":"2024-09-13T14:57:35.910479+0000","last_fullsized":"2024-09-13T14:57:35.910479+0000","mapping_epoch":449,"log_start":"460'5600","ondisk_log_start":"460'5600","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7552,"log_dups_size":0,"ondisk_log_size":7552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13152,"num_write_kb":513024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":591,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.832634+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:57:21.832634+0000","last_peered":"2024-09-13T14:57:21.832634+0000","last_clean":"2024-09-13T14:57:21.832634+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:57:21.832634+0000","last_undegraded":"2024-09-13T14:57:21.832634+0000","last_fullsized":"2024-09-13T14:57:21.832634+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":880,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.853965+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:57:26.853965+0000","last_peered":"2024-09-13T14:57:26.853965+0000","last_clean":"2024-09-13T14:57:26.853965+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:57:26.853965+0000","last_undegraded":"2024-09-13T14:57:26.853965+0000","last_fullsized":"2024-09-13T14:57:26.853965+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.829346+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:57:21.829346+0000","last_peered":"2024-09-13T14:57:21.829346+0000","last_clean":"2024-09-13T14:57:21.829346+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:57:21.829346+0000","last_undegraded":"2024-09-13T14:57:21.829346+0000","last_fullsized":"2024-09-13T14:57:21.829346+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"480'12992","reported_seq":13031,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.463291+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:57:38.463291+0000","last_peered":"2024-09-13T14:57:38.463291+0000","last_clean":"2024-09-13T14:57:38.463291+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:57:38.463291+0000","last_undegraded":"2024-09-13T14:57:38.463291+0000","last_fullsized":"2024-09-13T14:57:38.463291+0000","mapping_epoch":449,"log_start":"459'4400","ondisk_log_start":"459'4400","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8592,"log_dups_size":0,"ondisk_log_size":8592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173408256,"num_objects":2646,"num_object_clones":0,"num_object_copies":7938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12992,"num_write_kb":500416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":305,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824154+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:57:21.824154+0000","last_peered":"2024-09-13T14:57:21.824154+0000","last_clean":"2024-09-13T14:57:21.824154+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:57:21.824154+0000","last_undegraded":"2024-09-13T14:57:21.824154+0000","last_fullsized":"2024-09-13T14:57:21.824154+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":481,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830083+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:57:21.830083+0000","last_peered":"2024-09-13T14:57:21.830083+0000","last_clean":"2024-09-13T14:57:21.830083+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:57:21.830083+0000","last_undegraded":"2024-09-13T14:57:21.830083+0000","last_fullsized":"2024-09-13T14:57:21.830083+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":192,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.826783+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:57:21.826783+0000","last_peered":"2024-09-13T14:57:21.826783+0000","last_clean":"2024-09-13T14:57:21.826783+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:57:21.826783+0000","last_undegraded":"2024-09-13T14:57:21.826783+0000","last_fullsized":"2024-09-13T14:57:21.826783+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":682,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842586+0000","last_change":"2024-09-13T14:56:00.671941+0000","last_active":"2024-09-13T14:57:26.842586+0000","last_peered":"2024-09-13T14:57:26.842586+0000","last_clean":"2024-09-13T14:57:26.842586+0000","last_became_active":"2024-09-13T14:56:00.668042+0000","last_became_peered":"2024-09-13T14:56:00.668042+0000","last_unstale":"2024-09-13T14:57:26.842586+0000","last_undegraded":"2024-09-13T14:57:26.842586+0000","last_fullsized":"2024-09-13T14:57:26.842586+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.375774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.375774+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:18.945468+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"480'13029","reported_seq":13068,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.937292+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:57:37.937292+0000","last_peered":"2024-09-13T14:57:37.937292+0000","last_clean":"2024-09-13T14:57:37.937292+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:57:37.937292+0000","last_undegraded":"2024-09-13T14:57:37.937292+0000","last_fullsized":"2024-09-13T14:57:37.937292+0000","mapping_epoch":449,"log_start":"460'5967","ondisk_log_start":"460'5967","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178978816,"num_objects":2731,"num_object_clones":0,"num_object_copies":8193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13029,"num_write_kb":504320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.825612+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:57:21.825612+0000","last_peered":"2024-09-13T14:57:21.825612+0000","last_clean":"2024-09-13T14:57:21.825612+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:57:21.825612+0000","last_undegraded":"2024-09-13T14:57:21.825612+0000","last_fullsized":"2024-09-13T14:57:21.825612+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.827105+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:57:21.827105+0000","last_peered":"2024-09-13T14:57:21.827105+0000","last_clean":"2024-09-13T14:57:21.827105+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:57:21.827105+0000","last_undegraded":"2024-09-13T14:57:21.827105+0000","last_fullsized":"2024-09-13T14:57:21.827105+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"480'12912","reported_seq":12951,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.080217+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:57:38.080217+0000","last_peered":"2024-09-13T14:57:38.080217+0000","last_clean":"2024-09-13T14:57:38.080217+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:57:38.080217+0000","last_undegraded":"2024-09-13T14:57:38.080217+0000","last_fullsized":"2024-09-13T14:57:38.080217+0000","mapping_epoch":449,"log_start":"464'6735","ondisk_log_start":"464'6735","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174981120,"num_objects":2670,"num_object_clones":0,"num_object_copies":8010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12912,"num_write_kb":498624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":298,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824417+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:57:21.824417+0000","last_peered":"2024-09-13T14:57:21.824417+0000","last_clean":"2024-09-13T14:57:21.824417+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:57:21.824417+0000","last_undegraded":"2024-09-13T14:57:21.824417+0000","last_fullsized":"2024-09-13T14:57:21.824417+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":851,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850827+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:57:26.850827+0000","last_peered":"2024-09-13T14:57:26.850827+0000","last_clean":"2024-09-13T14:57:26.850827+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:57:26.850827+0000","last_undegraded":"2024-09-13T14:57:26.850827+0000","last_fullsized":"2024-09-13T14:57:26.850827+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":813,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.843576+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:57:26.843576+0000","last_peered":"2024-09-13T14:57:26.843576+0000","last_clean":"2024-09-13T14:57:26.843576+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:57:26.843576+0000","last_undegraded":"2024-09-13T14:57:26.843576+0000","last_fullsized":"2024-09-13T14:57:26.843576+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"480'13092","reported_seq":13130,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.968771+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:57:37.968771+0000","last_peered":"2024-09-13T14:57:37.968771+0000","last_clean":"2024-09-13T14:57:37.968771+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:57:37.968771+0000","last_undegraded":"2024-09-13T14:57:37.968771+0000","last_fullsized":"2024-09-13T14:57:37.968771+0000","mapping_epoch":449,"log_start":"461'6100","ondisk_log_start":"461'6100","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175898624,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13092,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":622,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.834369+0000","last_change":"2024-09-13T14:55:57.906683+0000","last_active":"2024-09-13T14:57:21.834369+0000","last_peered":"2024-09-13T14:57:21.834369+0000","last_clean":"2024-09-13T14:57:21.834369+0000","last_became_active":"2024-09-13T14:55:54.613915+0000","last_became_peered":"2024-09-13T14:55:54.613915+0000","last_unstale":"2024-09-13T14:57:21.834369+0000","last_undegraded":"2024-09-13T14:57:21.834369+0000","last_fullsized":"2024-09-13T14:57:21.834369+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.906630+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.906630+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:03.677753+0000","scrub_duration":0.001255616,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":1004,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824619+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:57:21.824619+0000","last_peered":"2024-09-13T14:57:21.824619+0000","last_clean":"2024-09-13T14:57:21.824619+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:57:21.824619+0000","last_undegraded":"2024-09-13T14:57:21.824619+0000","last_fullsized":"2024-09-13T14:57:21.824619+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":805,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.849943+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:57:26.849943+0000","last_peered":"2024-09-13T14:57:26.849943+0000","last_clean":"2024-09-13T14:57:26.849943+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:57:26.849943+0000","last_undegraded":"2024-09-13T14:57:26.849943+0000","last_fullsized":"2024-09-13T14:57:26.849943+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":446,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830632+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:57:21.830632+0000","last_peered":"2024-09-13T14:57:21.830632+0000","last_clean":"2024-09-13T14:57:21.830632+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:57:21.830632+0000","last_undegraded":"2024-09-13T14:57:21.830632+0000","last_fullsized":"2024-09-13T14:57:21.830632+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"480'13073","reported_seq":13147,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.556535+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:57:37.556535+0000","last_peered":"2024-09-13T14:57:37.556535+0000","last_clean":"2024-09-13T14:57:37.556535+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:57:37.556535+0000","last_undegraded":"2024-09-13T14:57:37.556535+0000","last_fullsized":"2024-09-13T14:57:37.556535+0000","mapping_epoch":449,"log_start":"460'5902","ondisk_log_start":"460'5902","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7171,"log_dups_size":0,"ondisk_log_size":7171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172556288,"num_objects":2633,"num_object_clones":0,"num_object_copies":7899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13073,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":679,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842759+0000","last_change":"2024-09-13T14:56:00.322316+0000","last_active":"2024-09-13T14:57:26.842759+0000","last_peered":"2024-09-13T14:57:26.842759+0000","last_clean":"2024-09-13T14:57:26.842759+0000","last_became_active":"2024-09-13T14:55:54.607589+0000","last_became_peered":"2024-09-13T14:55:54.607589+0000","last_unstale":"2024-09-13T14:57:26.842759+0000","last_undegraded":"2024-09-13T14:57:26.842759+0000","last_fullsized":"2024-09-13T14:57:26.842759+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.322230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.322230+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:10.503431+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":892,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.834318+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:57:21.834318+0000","last_peered":"2024-09-13T14:57:21.834318+0000","last_clean":"2024-09-13T14:57:21.834318+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:57:21.834318+0000","last_undegraded":"2024-09-13T14:57:21.834318+0000","last_fullsized":"2024-09-13T14:57:21.834318+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":136,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.866767+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:57:26.866767+0000","last_peered":"2024-09-13T14:57:26.866767+0000","last_clean":"2024-09-13T14:57:26.866767+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:57:26.866767+0000","last_undegraded":"2024-09-13T14:57:26.866767+0000","last_fullsized":"2024-09-13T14:57:26.866767+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":136,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.883915+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:57:26.883915+0000","last_peered":"2024-09-13T14:57:26.883915+0000","last_clean":"2024-09-13T14:57:26.883915+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:57:26.883915+0000","last_undegraded":"2024-09-13T14:57:26.883915+0000","last_fullsized":"2024-09-13T14:57:26.883915+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"480'13260","reported_seq":13299,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.480875+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:57:38.480875+0000","last_peered":"2024-09-13T14:57:38.480875+0000","last_clean":"2024-09-13T14:57:38.480875+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:57:38.480875+0000","last_undegraded":"2024-09-13T14:57:38.480875+0000","last_fullsized":"2024-09-13T14:57:38.480875+0000","mapping_epoch":449,"log_start":"459'4600","ondisk_log_start":"459'4600","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8660,"log_dups_size":0,"ondisk_log_size":8660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178126848,"num_objects":2718,"num_object_clones":0,"num_object_copies":8154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13260,"num_write_kb":511296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":651,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830501+0000","last_change":"2024-09-13T14:55:58.908251+0000","last_active":"2024-09-13T14:57:21.830501+0000","last_peered":"2024-09-13T14:57:21.830501+0000","last_clean":"2024-09-13T14:57:21.830501+0000","last_became_active":"2024-09-13T14:55:54.619910+0000","last_became_peered":"2024-09-13T14:55:54.619910+0000","last_unstale":"2024-09-13T14:57:21.830501+0000","last_undegraded":"2024-09-13T14:57:21.830501+0000","last_fullsized":"2024-09-13T14:57:21.830501+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.908152+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.908152+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:01.217177+0000","scrub_duration":0.0048598549999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":831,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850929+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:57:26.850929+0000","last_peered":"2024-09-13T14:57:26.850929+0000","last_clean":"2024-09-13T14:57:26.850929+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:57:26.850929+0000","last_undegraded":"2024-09-13T14:57:26.850929+0000","last_fullsized":"2024-09-13T14:57:26.850929+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":791,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.843230+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:57:26.843230+0000","last_peered":"2024-09-13T14:57:26.843230+0000","last_clean":"2024-09-13T14:57:26.843230+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:57:26.843230+0000","last_undegraded":"2024-09-13T14:57:26.843230+0000","last_fullsized":"2024-09-13T14:57:26.843230+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":249,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.825538+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:57:21.825538+0000","last_peered":"2024-09-13T14:57:21.825538+0000","last_clean":"2024-09-13T14:57:21.825538+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:57:21.825538+0000","last_undegraded":"2024-09-13T14:57:21.825538+0000","last_fullsized":"2024-09-13T14:57:21.825538+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"480'13112","reported_seq":13151,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.957904+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:57:37.957904+0000","last_peered":"2024-09-13T14:57:37.957904+0000","last_clean":"2024-09-13T14:57:37.957904+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:57:37.957904+0000","last_undegraded":"2024-09-13T14:57:37.957904+0000","last_fullsized":"2024-09-13T14:57:37.957904+0000","mapping_epoch":449,"log_start":"460'6100","ondisk_log_start":"460'6100","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169213952,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13112,"num_write_kb":502208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":338,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.827251+0000","last_change":"2024-09-13T14:56:00.644560+0000","last_active":"2024-09-13T14:57:21.827251+0000","last_peered":"2024-09-13T14:57:21.827251+0000","last_clean":"2024-09-13T14:57:21.827251+0000","last_became_active":"2024-09-13T14:55:54.621234+0000","last_became_peered":"2024-09-13T14:55:54.621234+0000","last_unstale":"2024-09-13T14:57:21.827251+0000","last_undegraded":"2024-09-13T14:57:21.827251+0000","last_fullsized":"2024-09-13T14:57:21.827251+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.644466+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.644466+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.375763+0000","scrub_duration":0.0093267220000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1095,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824503+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:57:21.824503+0000","last_peered":"2024-09-13T14:57:21.824503+0000","last_clean":"2024-09-13T14:57:21.824503+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:57:21.824503+0000","last_undegraded":"2024-09-13T14:57:21.824503+0000","last_fullsized":"2024-09-13T14:57:21.824503+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":198,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.826979+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:57:21.826979+0000","last_peered":"2024-09-13T14:57:21.826979+0000","last_clean":"2024-09-13T14:57:21.826979+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:57:21.826979+0000","last_undegraded":"2024-09-13T14:57:21.826979+0000","last_fullsized":"2024-09-13T14:57:21.826979+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":811,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842518+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:57:26.842518+0000","last_peered":"2024-09-13T14:57:26.842518+0000","last_clean":"2024-09-13T14:57:26.842518+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:57:26.842518+0000","last_undegraded":"2024-09-13T14:57:26.842518+0000","last_fullsized":"2024-09-13T14:57:26.842518+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"480'13220","reported_seq":13259,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.297014+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:57:38.297014+0000","last_peered":"2024-09-13T14:57:38.297014+0000","last_clean":"2024-09-13T14:57:38.297014+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:57:38.297014+0000","last_undegraded":"2024-09-13T14:57:38.297014+0000","last_fullsized":"2024-09-13T14:57:38.297014+0000","mapping_epoch":449,"log_start":"460'6042","ondisk_log_start":"460'6042","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179568640,"num_objects":2740,"num_object_clones":0,"num_object_copies":8220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13220,"num_write_kb":510720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":499,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830765+0000","last_change":"2024-09-13T14:55:59.724453+0000","last_active":"2024-09-13T14:57:21.830765+0000","last_peered":"2024-09-13T14:57:21.830765+0000","last_clean":"2024-09-13T14:57:21.830765+0000","last_became_active":"2024-09-13T14:55:54.613965+0000","last_became_peered":"2024-09-13T14:55:54.613965+0000","last_unstale":"2024-09-13T14:57:21.830765+0000","last_undegraded":"2024-09-13T14:57:21.830765+0000","last_fullsized":"2024-09-13T14:57:21.830765+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.723985+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.723985+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:11.061534+0000","scrub_duration":0.0085871660000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":425,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824822+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:57:21.824822+0000","last_peered":"2024-09-13T14:57:21.824822+0000","last_clean":"2024-09-13T14:57:21.824822+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:57:21.824822+0000","last_undegraded":"2024-09-13T14:57:21.824822+0000","last_fullsized":"2024-09-13T14:57:21.824822+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1474,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842681+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:57:26.842681+0000","last_peered":"2024-09-13T14:57:26.842681+0000","last_clean":"2024-09-13T14:57:26.842681+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:57:26.842681+0000","last_undegraded":"2024-09-13T14:57:26.842681+0000","last_fullsized":"2024-09-13T14:57:26.842681+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":792,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844197+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:57:26.844197+0000","last_peered":"2024-09-13T14:57:26.844197+0000","last_clean":"2024-09-13T14:57:26.844197+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:57:26.844197+0000","last_undegraded":"2024-09-13T14:57:26.844197+0000","last_fullsized":"2024-09-13T14:57:26.844197+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"480'12919","reported_seq":12959,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.954848+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:57:37.954848+0000","last_peered":"2024-09-13T14:57:37.954848+0000","last_clean":"2024-09-13T14:57:37.954848+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:57:37.954848+0000","last_undegraded":"2024-09-13T14:57:37.954848+0000","last_fullsized":"2024-09-13T14:57:37.954848+0000","mapping_epoch":449,"log_start":"460'5883","ondisk_log_start":"460'5883","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172228632,"num_objects":2629,"num_object_clones":0,"num_object_copies":7887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2629,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12919,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":519,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830156+0000","last_change":"2024-09-13T14:56:00.740088+0000","last_active":"2024-09-13T14:57:21.830156+0000","last_peered":"2024-09-13T14:57:21.830156+0000","last_clean":"2024-09-13T14:57:21.830156+0000","last_became_active":"2024-09-13T14:55:54.621088+0000","last_became_peered":"2024-09-13T14:55:54.621088+0000","last_unstale":"2024-09-13T14:57:21.830156+0000","last_undegraded":"2024-09-13T14:57:21.830156+0000","last_fullsized":"2024-09-13T14:57:21.830156+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.739931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.739931+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.959219+0000","scrub_duration":0.0093782959999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":801,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842597+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:57:26.842597+0000","last_peered":"2024-09-13T14:57:26.842597+0000","last_clean":"2024-09-13T14:57:26.842597+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:57:26.842597+0000","last_undegraded":"2024-09-13T14:57:26.842597+0000","last_fullsized":"2024-09-13T14:57:26.842597+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":896,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831344+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:57:21.831344+0000","last_peered":"2024-09-13T14:57:21.831344+0000","last_clean":"2024-09-13T14:57:21.831344+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:57:21.831344+0000","last_undegraded":"2024-09-13T14:57:21.831344+0000","last_fullsized":"2024-09-13T14:57:21.831344+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":136,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.869490+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:57:26.869490+0000","last_peered":"2024-09-13T14:57:26.869490+0000","last_clean":"2024-09-13T14:57:26.869490+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:57:26.869490+0000","last_undegraded":"2024-09-13T14:57:26.869490+0000","last_fullsized":"2024-09-13T14:57:26.869490+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"480'13226","reported_seq":13265,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:38.129635+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:57:38.129635+0000","last_peered":"2024-09-13T14:57:38.129635+0000","last_clean":"2024-09-13T14:57:38.129635+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:57:38.129635+0000","last_undegraded":"2024-09-13T14:57:38.129635+0000","last_fullsized":"2024-09-13T14:57:38.129635+0000","mapping_epoch":449,"log_start":"465'7056","ondisk_log_start":"465'7056","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13226,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":198,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.827192+0000","last_change":"2024-09-13T14:56:00.807834+0000","last_active":"2024-09-13T14:57:21.827192+0000","last_peered":"2024-09-13T14:57:21.827192+0000","last_clean":"2024-09-13T14:57:21.827192+0000","last_became_active":"2024-09-13T14:55:54.609192+0000","last_became_peered":"2024-09-13T14:55:54.609192+0000","last_unstale":"2024-09-13T14:57:21.827192+0000","last_undegraded":"2024-09-13T14:57:21.827192+0000","last_fullsized":"2024-09-13T14:57:21.827192+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.807561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.807561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:44.901235+0000","scrub_duration":0.012160298999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":859,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.857549+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:57:26.857549+0000","last_peered":"2024-09-13T14:57:26.857549+0000","last_clean":"2024-09-13T14:57:26.857549+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:57:26.857549+0000","last_undegraded":"2024-09-13T14:57:26.857549+0000","last_fullsized":"2024-09-13T14:57:26.857549+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1177,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.843315+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:57:26.843315+0000","last_peered":"2024-09-13T14:57:26.843315+0000","last_clean":"2024-09-13T14:57:26.843315+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:57:26.843315+0000","last_undegraded":"2024-09-13T14:57:26.843315+0000","last_fullsized":"2024-09-13T14:57:26.843315+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":849,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850654+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:57:26.850654+0000","last_peered":"2024-09-13T14:57:26.850654+0000","last_clean":"2024-09-13T14:57:26.850654+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:57:26.850654+0000","last_undegraded":"2024-09-13T14:57:26.850654+0000","last_fullsized":"2024-09-13T14:57:26.850654+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"480'13223","reported_seq":13262,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:37.920055+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:57:37.920055+0000","last_peered":"2024-09-13T14:57:37.920055+0000","last_clean":"2024-09-13T14:57:37.920055+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:57:37.920055+0000","last_undegraded":"2024-09-13T14:57:37.920055+0000","last_fullsized":"2024-09-13T14:57:37.920055+0000","mapping_epoch":449,"log_start":"461'6184","ondisk_log_start":"461'6184","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13223,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":710,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851770+0000","last_change":"2024-09-13T14:56:00.000186+0000","last_active":"2024-09-13T14:57:26.851770+0000","last_peered":"2024-09-13T14:57:26.851770+0000","last_clean":"2024-09-13T14:57:26.851770+0000","last_became_active":"2024-09-13T14:55:54.611253+0000","last_became_peered":"2024-09-13T14:55:54.611253+0000","last_unstale":"2024-09-13T14:57:26.851770+0000","last_undegraded":"2024-09-13T14:57:26.851770+0000","last_fullsized":"2024-09-13T14:57:26.851770+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.999712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.999712+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:46.770712+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":140,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.884018+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:57:26.884018+0000","last_peered":"2024-09-13T14:57:26.884018+0000","last_clean":"2024-09-13T14:57:26.884018+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:57:26.884018+0000","last_undegraded":"2024-09-13T14:57:26.884018+0000","last_fullsized":"2024-09-13T14:57:26.884018+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":898,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830957+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:57:21.830957+0000","last_peered":"2024-09-13T14:57:21.830957+0000","last_clean":"2024-09-13T14:57:21.830957+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:57:21.830957+0000","last_undegraded":"2024-09-13T14:57:21.830957+0000","last_fullsized":"2024-09-13T14:57:21.830957+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":192,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.825726+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:57:21.825726+0000","last_peered":"2024-09-13T14:57:21.825726+0000","last_clean":"2024-09-13T14:57:21.825726+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:57:21.825726+0000","last_undegraded":"2024-09-13T14:57:21.825726+0000","last_fullsized":"2024-09-13T14:57:21.825726+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.827154+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:57:21.827154+0000","last_peered":"2024-09-13T14:57:21.827154+0000","last_clean":"2024-09-13T14:57:21.827154+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:57:21.827154+0000","last_undegraded":"2024-09-13T14:57:21.827154+0000","last_fullsized":"2024-09-13T14:57:21.827154+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":188,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.829230+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:57:21.829230+0000","last_peered":"2024-09-13T14:57:21.829230+0000","last_clean":"2024-09-13T14:57:21.829230+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:57:21.829230+0000","last_undegraded":"2024-09-13T14:57:21.829230+0000","last_fullsized":"2024-09-13T14:57:21.829230+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":788,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844004+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:57:26.844004+0000","last_peered":"2024-09-13T14:57:26.844004+0000","last_clean":"2024-09-13T14:57:26.844004+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:57:26.844004+0000","last_undegraded":"2024-09-13T14:57:26.844004+0000","last_fullsized":"2024-09-13T14:57:26.844004+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":479,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.831149+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:57:21.831149+0000","last_peered":"2024-09-13T14:57:21.831149+0000","last_clean":"2024-09-13T14:57:21.831149+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:57:21.831149+0000","last_undegraded":"2024-09-13T14:57:21.831149+0000","last_fullsized":"2024-09-13T14:57:21.831149+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":293,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830793+0000","last_change":"2024-09-13T14:56:06.913381+0000","last_active":"2024-09-13T14:57:21.830793+0000","last_peered":"2024-09-13T14:57:21.830793+0000","last_clean":"2024-09-13T14:57:21.830793+0000","last_became_active":"2024-09-13T14:56:00.679102+0000","last_became_peered":"2024-09-13T14:56:00.679102+0000","last_unstale":"2024-09-13T14:57:21.830793+0000","last_undegraded":"2024-09-13T14:57:21.830793+0000","last_fullsized":"2024-09-13T14:57:21.830793+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:06.913154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:06.913154+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:14.474904+0000","scrub_duration":0.0026651819999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":445,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824680+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:57:21.824680+0000","last_peered":"2024-09-13T14:57:21.824680+0000","last_clean":"2024-09-13T14:57:21.824680+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:57:21.824680+0000","last_undegraded":"2024-09-13T14:57:21.824680+0000","last_fullsized":"2024-09-13T14:57:21.824680+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":793,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850195+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:57:26.850195+0000","last_peered":"2024-09-13T14:57:26.850195+0000","last_clean":"2024-09-13T14:57:26.850195+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:57:26.850195+0000","last_undegraded":"2024-09-13T14:57:26.850195+0000","last_fullsized":"2024-09-13T14:57:26.850195+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.866653+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:57:26.866653+0000","last_peered":"2024-09-13T14:57:26.866653+0000","last_clean":"2024-09-13T14:57:26.866653+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:57:26.866653+0000","last_undegraded":"2024-09-13T14:57:26.866653+0000","last_fullsized":"2024-09-13T14:57:26.866653+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":849,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850560+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:57:26.850560+0000","last_peered":"2024-09-13T14:57:26.850560+0000","last_clean":"2024-09-13T14:57:26.850560+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:57:26.850560+0000","last_undegraded":"2024-09-13T14:57:26.850560+0000","last_fullsized":"2024-09-13T14:57:26.850560+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":477,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.830859+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:57:21.830859+0000","last_peered":"2024-09-13T14:57:21.830859+0000","last_clean":"2024-09-13T14:57:21.830859+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:57:21.830859+0000","last_undegraded":"2024-09-13T14:57:21.830859+0000","last_fullsized":"2024-09-13T14:57:21.830859+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":880,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850735+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:57:26.850735+0000","last_peered":"2024-09-13T14:57:26.850735+0000","last_clean":"2024-09-13T14:57:26.850735+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:57:26.850735+0000","last_undegraded":"2024-09-13T14:57:26.850735+0000","last_fullsized":"2024-09-13T14:57:26.850735+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":136,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.884140+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:57:26.884140+0000","last_peered":"2024-09-13T14:57:26.884140+0000","last_clean":"2024-09-13T14:57:26.884140+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:57:26.884140+0000","last_undegraded":"2024-09-13T14:57:26.884140+0000","last_fullsized":"2024-09-13T14:57:26.884140+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":813,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842398+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:57:26.842398+0000","last_peered":"2024-09-13T14:57:26.842398+0000","last_clean":"2024-09-13T14:57:26.842398+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:57:26.842398+0000","last_undegraded":"2024-09-13T14:57:26.842398+0000","last_fullsized":"2024-09-13T14:57:26.842398+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":862,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851015+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:57:26.851015+0000","last_peered":"2024-09-13T14:57:26.851015+0000","last_clean":"2024-09-13T14:57:26.851015+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:57:26.851015+0000","last_undegraded":"2024-09-13T14:57:26.851015+0000","last_fullsized":"2024-09-13T14:57:26.851015+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":177,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.826960+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:57:21.826960+0000","last_peered":"2024-09-13T14:57:21.826960+0000","last_clean":"2024-09-13T14:57:21.826960+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:57:21.826960+0000","last_undegraded":"2024-09-13T14:57:21.826960+0000","last_fullsized":"2024-09-13T14:57:21.826960+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":759,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842812+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:57:26.842812+0000","last_peered":"2024-09-13T14:57:26.842812+0000","last_clean":"2024-09-13T14:57:26.842812+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:57:26.842812+0000","last_undegraded":"2024-09-13T14:57:26.842812+0000","last_fullsized":"2024-09-13T14:57:26.842812+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":135,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.869574+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:57:26.869574+0000","last_peered":"2024-09-13T14:57:26.869574+0000","last_clean":"2024-09-13T14:57:26.869574+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:57:26.869574+0000","last_undegraded":"2024-09-13T14:57:26.869574+0000","last_fullsized":"2024-09-13T14:57:26.869574+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.826121+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:57:21.826121+0000","last_peered":"2024-09-13T14:57:21.826121+0000","last_clean":"2024-09-13T14:57:21.826121+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:57:21.826121+0000","last_undegraded":"2024-09-13T14:57:21.826121+0000","last_fullsized":"2024-09-13T14:57:21.826121+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":445,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824688+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:57:21.824688+0000","last_peered":"2024-09-13T14:57:21.824688+0000","last_clean":"2024-09-13T14:57:21.824688+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:57:21.824688+0000","last_undegraded":"2024-09-13T14:57:21.824688+0000","last_fullsized":"2024-09-13T14:57:21.824688+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.826875+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:57:21.826875+0000","last_peered":"2024-09-13T14:57:21.826875+0000","last_clean":"2024-09-13T14:57:21.826875+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:57:21.826875+0000","last_undegraded":"2024-09-13T14:57:21.826875+0000","last_fullsized":"2024-09-13T14:57:21.826875+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":411,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.824965+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:57:21.824965+0000","last_peered":"2024-09-13T14:57:21.824965+0000","last_clean":"2024-09-13T14:57:21.824965+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:57:21.824965+0000","last_undegraded":"2024-09-13T14:57:21.824965+0000","last_fullsized":"2024-09-13T14:57:21.824965+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":872,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850507+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:57:26.850507+0000","last_peered":"2024-09-13T14:57:26.850507+0000","last_clean":"2024-09-13T14:57:26.850507+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:57:26.850507+0000","last_undegraded":"2024-09-13T14:57:26.850507+0000","last_fullsized":"2024-09-13T14:57:26.850507+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":846,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.850174+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:57:26.850174+0000","last_peered":"2024-09-13T14:57:26.850174+0000","last_clean":"2024-09-13T14:57:26.850174+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:57:26.850174+0000","last_undegraded":"2024-09-13T14:57:26.850174+0000","last_fullsized":"2024-09-13T14:57:26.850174+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":178,"reported_epoch":479,"state":"active+clean","last_fresh":"2024-09-13T14:57:21.829143+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:57:21.829143+0000","last_peered":"2024-09-13T14:57:21.829143+0000","last_clean":"2024-09-13T14:57:21.829143+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:57:21.829143+0000","last_undegraded":"2024-09-13T14:57:21.829143+0000","last_fullsized":"2024-09-13T14:57:21.829143+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":142,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.866551+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:57:26.866551+0000","last_peered":"2024-09-13T14:57:26.866551+0000","last_clean":"2024-09-13T14:57:26.866551+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:57:26.866551+0000","last_undegraded":"2024-09-13T14:57:26.866551+0000","last_fullsized":"2024-09-13T14:57:26.866551+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2816671768,"num_objects":42980,"num_object_clones":0,"num_object_copies":128940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42980,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209624,"num_write_kb":8083265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8494460928,"data_stored":8494448712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116690,"ondisk_log_size":116690,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398058,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588404,"kb_used_data":696556,"kb_used_omap":1,"kb_used_meta":891838,"kb_avail":92156748,"statfs":{"total":95995035648,"available":94368509952,"internally_reserved":0,"allocated":713273344,"data_stored":710747106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913242456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594286,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2101800,"kb_used_data":1207328,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":91643352,"statfs":{"total":95995035648,"available":93842792448,"internally_reserved":0,"allocated":1236303872,"data_stored":1233789244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665740,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1765920,"kb_used_data":848216,"kb_used_omap":9,"kb_used_meta":917686,"kb_avail":91979232,"statfs":{"total":95995035648,"available":94186733568,"internally_reserved":0,"allocated":868573184,"data_stored":866063031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":939710664},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986742,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2313012,"kb_used_data":1409004,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":91432140,"statfs":{"total":95995035648,"available":93626511360,"internally_reserved":0,"allocated":1442820096,"data_stored":1440293812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697302,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1977696,"kb_used_data":860440,"kb_used_omap":4,"kb_used_meta":1117243,"kb_avail":91767456,"statfs":{"total":95995035648,"available":93969874944,"internally_reserved":0,"allocated":881090560,"data_stored":878580488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1144056907},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860828,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1964132,"kb_used_data":1097628,"kb_used_omap":8,"kb_used_meta":866487,"kb_avail":91781020,"statfs":{"total":95995035648,"available":93983764480,"internally_reserved":0,"allocated":1123971072,"data_stored":1121457418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":887283532},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020499,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1201948,"kb_used_data":562516,"kb_used_omap":9,"kb_used_meta":639286,"kb_avail":92543204,"statfs":{"total":95995035648,"available":94764240896,"internally_reserved":0,"allocated":576016384,"data_stored":573509785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":654629701},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187890,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2728160,"kb_used_data":1744344,"kb_used_omap":7,"kb_used_meta":983736,"kb_avail":91016992,"statfs":{"total":95995035648,"available":93201399808,"internally_reserved":0,"allocated":1786208256,"data_stored":1783697851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":1007345995},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1763512320,"data_stored":1763508248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":560201728,"data_stored":560201728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1112145920,"data_stored":1112145920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":857276416,"data_stored":857276416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1412960256,"data_stored":1412956184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":858980352,"data_stored":858980352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1227165696,"data_stored":1227161624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":702218240,"data_stored":702218240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:57:40.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:57:40.745 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:40.777 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:40.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[103001]: pgmap v1585: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:57:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[103001]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[103001]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[107556]: pgmap v1585: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[107556]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:57:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:40 smithi017 ceph-mon[107556]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-mon[90992]: pgmap v1585: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-09-13T14:57:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-mon[90992]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:57:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:40 smithi189 ceph-mon[90992]: from='client.47165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:41.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.065 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:41.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:41.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:41.316 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.354 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.455 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:41.488 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:41.590 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.625 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.727 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:41.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:41.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:41.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:42.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.348 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:42.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:42.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:42.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:42.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.630 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:42.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:42.752 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:42 smithi017 ceph-mon[103001]: pgmap v1587: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:57:42.752 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:42.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:42 smithi017 ceph-mon[107556]: pgmap v1587: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:57:42.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:42.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:42.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:43.002 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:43.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:42 smithi189 ceph-mon[90992]: pgmap v1587: 109 pgs: 109 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-13T14:57:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:43.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.073 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:57:43.073 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.085 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:43.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.103 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:57:43.104 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:43.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:57:43.135 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:43.170 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:57:43.171 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:43.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.206 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:57:43.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:43.218 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:43.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:57:43.241 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:43.274 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:57:43.275 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:43.307 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:57:43.308 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:43.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:43.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.639 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:43.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:43.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.877 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3193727140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:43.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3193727140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:43.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:57:44.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3193727140' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:44.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.166 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:44.207 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:44.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:44.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:44.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:44.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:44.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:44.609 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.655 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:44 smithi017 ceph-mon[103001]: pgmap v1588: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:57:44.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:44 smithi017 ceph-mon[107556]: pgmap v1588: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:57:44.756 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:44.795 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:44.897 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.925 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:44.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:45.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:44 smithi189 ceph-mon[90992]: pgmap v1588: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-09-13T14:57:45.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:45.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:45.211 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:45.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:45.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:45.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:45.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:45.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:45.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:45.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.764 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:45.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:46.004 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.040 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:46.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:46.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:46.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:46.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:46.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.471 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:46.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:46.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:46.745 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:46.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:46.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:46.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:46 smithi017 ceph-mon[103001]: pgmap v1589: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:57:46.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:46 smithi017 ceph-mon[107556]: pgmap v1589: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:218129 10.3 e481/449 10:c13f843f:::benchmark_data_smithi017_372078_object91825:head v 481'13473, mlcod=481'13471)", 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:57:46.377726+0000", 2024-09-13T14:57:46.960 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0030005800000000001, 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030292840000000001, 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377726+0000", 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377726+0000", 2024-09-13T14:57:46.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377727+0000", 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2860000000000001e-06 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377734+0000", 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3170000000000004e-06 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:57:46.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377754+0000", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9349000000000001e-05 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377762+0000", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6510000000000007e-06 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.377961+0000", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019877999999999999 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:57:46.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378022+0000", 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1124999999999995e-05 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:218130 10.f e481/449 10:f9a22121:::benchmark_data_smithi017_372078_object91826:head v 481'13753, mlcod=481'13751)", 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:57:46.378101+0000", 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0026257400000000001, 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027134350000000002, 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:57:46.964 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378101+0000", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378101+0000", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378101+0000", 2024-09-13T14:57:46.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3600000000000003e-07 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378107+0000", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378118+0000", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1619e-05 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378124+0000", 2024-09-13T14:57:46.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1380000000000004e-06 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378286+0000", 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016133400000000001 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378339+0000", 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3752999999999998e-05 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:218127 10.4 e481/449 10:20a55b2e:::benchmark_data_smithi017_372078_object91823:head v 481'13716, mlcod=481'13714)", 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:57:46.378336+0000", 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0023906159999999999, 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025289610000000001, 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378336+0000", 2024-09-13T14:57:46.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378336+0000", 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378337+0000", 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3300000000000001e-07 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:57:46.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378343+0000", 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.584e-06 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378358+0000", 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4885999999999999e-05 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378365+0000", 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3200000000000002e-06 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378444+0000", 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8281999999999994e-05 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:57:46.378465+0000", 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1764e-05 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.971 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:57:46.972 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-13T14:57:46.972 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 3 2024-09-13T14:57:46.972 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:57:46.984 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:47.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:46 smithi189 ceph-mon[90992]: pgmap v1589: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T14:57:47.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.021 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:47.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:47.122 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:47.156 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:47.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:47.290 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:47.314 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:47.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:47.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.624 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:47] "GET /metrics HTTP/1.1" 200 36719 "" "Prometheus/2.51.0" 2024-09-13T14:57:47.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:47.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:57:47.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:47.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:47.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:48.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.143 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.171 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:48.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:48.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:48.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:48.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.572 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.609 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:48.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:48.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:48.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:48.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:48 smithi017 ceph-mon[103001]: pgmap v1590: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:57:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:48 smithi017 ceph-mon[107556]: pgmap v1590: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:57:48.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:48 smithi189 ceph-mon[90992]: pgmap v1590: 109 pgs: 109 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:57:49.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:49.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:49.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:49.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:49.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.409 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.441 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:49.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:49.542 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.575 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:49.677 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:49.712 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:49.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:49.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:49.955 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:50.093 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.136 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:50.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:50.220 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:50.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.260 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:57:50.262 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:57:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:50.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:50.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:57:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:50.562 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.598 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:50.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:50.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:50.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:50 smithi017 ceph-mon[103001]: pgmap v1591: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:57:50.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:50 smithi017 ceph-mon[107556]: pgmap v1591: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:57:50.994 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:50 smithi189 ceph-mon[90992]: pgmap v1591: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:57:51.035 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:51.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:51.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:51.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:51.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:51.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:51.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:51.600 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1591,"stamp":"2024-09-13T14:57:49.479047+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2163152869,"num_objects":32963,"num_object_clones":0,"num_object_copies":98889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32963,"num_whiteouts":0,"num_read":5969,"num_read_kb":12982,"num_write":220576,"num_write_kb":8095888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":103,"num_bytes_recovered":8721938,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119967,"ondisk_log_size":119967,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13780432,"kb_used_data":6514192,"kb_used_omap":44,"kb_used_meta":7266067,"kb_avail":736180784,"statfs":{"total":767960285184,"available":753849122816,"internally_reserved":0,"allocated":6670532608,"data_stored":6650382943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7440452855},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,5,8,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-658309120,"num_objects":-10045,"num_object_clones":0,"num_object_copies":-30135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10045,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005934"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":297,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561349+0000","last_change":"2024-09-13T14:56:00.831620+0000","last_active":"2024-09-13T14:57:39.561349+0000","last_peered":"2024-09-13T14:57:39.561349+0000","last_clean":"2024-09-13T14:57:39.561349+0000","last_became_active":"2024-09-13T14:56:00.676063+0000","last_became_peered":"2024-09-13T14:56:00.676063+0000","last_unstale":"2024-09-13T14:57:39.561349+0000","last_undegraded":"2024-09-13T14:57:39.561349+0000","last_fullsized":"2024-09-13T14:57:39.561349+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.831575+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.831575+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.735233+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":851,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.565986+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:57:39.565986+0000","last_peered":"2024-09-13T14:57:39.565986+0000","last_clean":"2024-09-13T14:57:39.565986+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:57:39.565986+0000","last_undegraded":"2024-09-13T14:57:39.565986+0000","last_fullsized":"2024-09-13T14:57:39.565986+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":479,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851538+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:57:26.851538+0000","last_peered":"2024-09-13T14:57:26.851538+0000","last_clean":"2024-09-13T14:57:26.851538+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:57:26.851538+0000","last_undegraded":"2024-09-13T14:57:26.851538+0000","last_fullsized":"2024-09-13T14:57:26.851538+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":34,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.841873+0000","last_change":"2024-09-13T14:56:04.574886+0000","last_active":"2024-09-13T14:57:26.841873+0000","last_peered":"2024-09-13T14:57:26.841873+0000","last_clean":"2024-09-13T14:57:26.841873+0000","last_became_active":"2024-09-13T14:56:00.674692+0000","last_became_peered":"2024-09-13T14:56:00.674692+0000","last_unstale":"2024-09-13T14:57:26.841873+0000","last_undegraded":"2024-09-13T14:57:26.841873+0000","last_fullsized":"2024-09-13T14:57:26.841873+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.574787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.574787+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:30.673194+0000","scrub_duration":0.0020770139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":408,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842339+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:57:26.842339+0000","last_peered":"2024-09-13T14:57:26.842339+0000","last_clean":"2024-09-13T14:57:26.842339+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:57:26.842339+0000","last_undegraded":"2024-09-13T14:57:26.842339+0000","last_fullsized":"2024-09-13T14:57:26.842339+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":137,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.568840+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:57:39.568840+0000","last_peered":"2024-09-13T14:57:39.568840+0000","last_clean":"2024-09-13T14:57:39.568840+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:57:39.568840+0000","last_undegraded":"2024-09-13T14:57:39.568840+0000","last_fullsized":"2024-09-13T14:57:39.568840+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":687,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553284+0000","last_change":"2024-09-13T14:56:01.351509+0000","last_active":"2024-09-13T14:57:39.553284+0000","last_peered":"2024-09-13T14:57:39.553284+0000","last_clean":"2024-09-13T14:57:39.553284+0000","last_became_active":"2024-09-13T14:56:00.675249+0000","last_became_peered":"2024-09-13T14:56:00.675249+0000","last_unstale":"2024-09-13T14:57:39.553284+0000","last_undegraded":"2024-09-13T14:57:39.553284+0000","last_fullsized":"2024-09-13T14:57:39.553284+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.351298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.351298+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:23.442259+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":137,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561553+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:57:39.561553+0000","last_peered":"2024-09-13T14:57:39.561553+0000","last_clean":"2024-09-13T14:57:39.561553+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:57:39.561553+0000","last_undegraded":"2024-09-13T14:57:39.561553+0000","last_fullsized":"2024-09-13T14:57:39.561553+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":448,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851420+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:57:26.851420+0000","last_peered":"2024-09-13T14:57:26.851420+0000","last_clean":"2024-09-13T14:57:26.851420+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:57:26.851420+0000","last_undegraded":"2024-09-13T14:57:26.851420+0000","last_fullsized":"2024-09-13T14:57:26.851420+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":34,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.841651+0000","last_change":"2024-09-13T14:56:01.635272+0000","last_active":"2024-09-13T14:57:26.841651+0000","last_peered":"2024-09-13T14:57:26.841651+0000","last_clean":"2024-09-13T14:57:26.841651+0000","last_became_active":"2024-09-13T14:56:00.667926+0000","last_became_peered":"2024-09-13T14:56:00.667926+0000","last_unstale":"2024-09-13T14:57:26.841651+0000","last_undegraded":"2024-09-13T14:57:26.841651+0000","last_fullsized":"2024-09-13T14:57:26.841651+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.635189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.635189+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:28.334884+0000","scrub_duration":0.010355512000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":850,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.566914+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:57:39.566914+0000","last_peered":"2024-09-13T14:57:39.566914+0000","last_clean":"2024-09-13T14:57:39.566914+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:57:39.566914+0000","last_undegraded":"2024-09-13T14:57:39.566914+0000","last_fullsized":"2024-09-13T14:57:39.566914+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":510,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.843997+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:57:26.843997+0000","last_peered":"2024-09-13T14:57:26.843997+0000","last_clean":"2024-09-13T14:57:26.843997+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:57:26.843997+0000","last_undegraded":"2024-09-13T14:57:26.843997+0000","last_fullsized":"2024-09-13T14:57:26.843997+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":34,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851641+0000","last_change":"2024-09-13T14:56:00.915923+0000","last_active":"2024-09-13T14:57:26.851641+0000","last_peered":"2024-09-13T14:57:26.851641+0000","last_clean":"2024-09-13T14:57:26.851641+0000","last_became_active":"2024-09-13T14:56:00.671638+0000","last_became_peered":"2024-09-13T14:56:00.671638+0000","last_unstale":"2024-09-13T14:57:26.851641+0000","last_undegraded":"2024-09-13T14:57:26.851641+0000","last_fullsized":"2024-09-13T14:57:26.851641+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.915835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.915835+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:52.043909+0000","scrub_duration":0.010872074000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"481'13623","reported_seq":13660,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.490197+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:57:48.490197+0000","last_peered":"2024-09-13T14:57:48.490197+0000","last_clean":"2024-09-13T14:57:48.490197+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:57:48.490197+0000","last_undegraded":"2024-09-13T14:57:48.490197+0000","last_fullsized":"2024-09-13T14:57:48.490197+0000","mapping_epoch":449,"log_start":"459'5000","ondisk_log_start":"459'5000","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8623,"log_dups_size":0,"ondisk_log_size":8623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13623,"num_write_kb":500736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":502,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851932+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:57:26.851932+0000","last_peered":"2024-09-13T14:57:26.851932+0000","last_clean":"2024-09-13T14:57:26.851932+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:57:26.851932+0000","last_undegraded":"2024-09-13T14:57:26.851932+0000","last_fullsized":"2024-09-13T14:57:26.851932+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":393,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.841648+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:57:26.841648+0000","last_peered":"2024-09-13T14:57:26.841648+0000","last_clean":"2024-09-13T14:57:26.841648+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:57:26.841648+0000","last_undegraded":"2024-09-13T14:57:26.841648+0000","last_fullsized":"2024-09-13T14:57:26.841648+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":720,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.565849+0000","last_change":"2024-09-13T14:56:00.697476+0000","last_active":"2024-09-13T14:57:39.565849+0000","last_peered":"2024-09-13T14:57:39.565849+0000","last_clean":"2024-09-13T14:57:39.565849+0000","last_became_active":"2024-09-13T14:56:00.680491+0000","last_became_peered":"2024-09-13T14:56:00.680491+0000","last_unstale":"2024-09-13T14:57:39.565849+0000","last_undegraded":"2024-09-13T14:57:39.565849+0000","last_fullsized":"2024-09-13T14:57:39.565849+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:56.869434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:56.869434+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:32.392000+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"481'13937","reported_seq":13977,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.953523+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:57:47.953523+0000","last_peered":"2024-09-13T14:57:47.953523+0000","last_clean":"2024-09-13T14:57:47.953523+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:57:47.953523+0000","last_undegraded":"2024-09-13T14:57:47.953523+0000","last_fullsized":"2024-09-13T14:57:47.953523+0000","mapping_epoch":449,"log_start":"463'6876","ondisk_log_start":"463'6876","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129433600,"num_objects":1975,"num_object_clones":0,"num_object_copies":5925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13937,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":492,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851780+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:57:26.851780+0000","last_peered":"2024-09-13T14:57:26.851780+0000","last_clean":"2024-09-13T14:57:26.851780+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:57:26.851780+0000","last_undegraded":"2024-09-13T14:57:26.851780+0000","last_fullsized":"2024-09-13T14:57:26.851780+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846594+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:57:26.846594+0000","last_peered":"2024-09-13T14:57:26.846594+0000","last_clean":"2024-09-13T14:57:26.846594+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:57:26.846594+0000","last_undegraded":"2024-09-13T14:57:26.846594+0000","last_fullsized":"2024-09-13T14:57:26.846594+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":509,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844312+0000","last_change":"2024-09-13T14:56:02.698903+0000","last_active":"2024-09-13T14:57:26.844312+0000","last_peered":"2024-09-13T14:57:26.844312+0000","last_clean":"2024-09-13T14:57:26.844312+0000","last_became_active":"2024-09-13T14:56:00.691242+0000","last_became_peered":"2024-09-13T14:56:00.691242+0000","last_unstale":"2024-09-13T14:57:26.844312+0000","last_undegraded":"2024-09-13T14:57:26.844312+0000","last_fullsized":"2024-09-13T14:57:26.844312+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.698817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.698817+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:11.424244+0000","scrub_duration":0.0077410969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"481'13798","reported_seq":13874,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.568154+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:57:47.568154+0000","last_peered":"2024-09-13T14:57:47.568154+0000","last_clean":"2024-09-13T14:57:47.568154+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:57:47.568154+0000","last_undegraded":"2024-09-13T14:57:47.568154+0000","last_fullsized":"2024-09-13T14:57:47.568154+0000","mapping_epoch":449,"log_start":"461'6556","ondisk_log_start":"461'6556","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13798,"num_write_kb":508480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":446,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851901+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:57:26.851901+0000","last_peered":"2024-09-13T14:57:26.851901+0000","last_clean":"2024-09-13T14:57:26.851901+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:57:26.851901+0000","last_undegraded":"2024-09-13T14:57:26.851901+0000","last_fullsized":"2024-09-13T14:57:26.851901+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":461,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844106+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:57:26.844106+0000","last_peered":"2024-09-13T14:57:26.844106+0000","last_clean":"2024-09-13T14:57:26.844106+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:57:26.844106+0000","last_undegraded":"2024-09-13T14:57:26.844106+0000","last_fullsized":"2024-09-13T14:57:26.844106+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":34,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846460+0000","last_change":"2024-09-13T14:56:05.805624+0000","last_active":"2024-09-13T14:57:26.846460+0000","last_peered":"2024-09-13T14:57:26.846460+0000","last_clean":"2024-09-13T14:57:26.846460+0000","last_became_active":"2024-09-13T14:56:00.676524+0000","last_became_peered":"2024-09-13T14:56:00.676524+0000","last_unstale":"2024-09-13T14:57:26.846460+0000","last_undegraded":"2024-09-13T14:57:26.846460+0000","last_fullsized":"2024-09-13T14:57:26.846460+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:05.805551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:05.805551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:13.686450+0000","scrub_duration":0.028580083999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.5","version":"481'13505","reported_seq":13581,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.634241+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:57:47.634241+0000","last_peered":"2024-09-13T14:57:47.634241+0000","last_clean":"2024-09-13T14:57:47.634241+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:57:47.634241+0000","last_undegraded":"2024-09-13T14:57:47.634241+0000","last_fullsized":"2024-09-13T14:57:47.634241+0000","mapping_epoch":449,"log_start":"461'6333","ondisk_log_start":"461'6333","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133890048,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13505,"num_write_kb":497536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":409,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842103+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:57:26.842103+0000","last_peered":"2024-09-13T14:57:26.842103+0000","last_clean":"2024-09-13T14:57:26.842103+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:57:26.842103+0000","last_undegraded":"2024-09-13T14:57:26.842103+0000","last_fullsized":"2024-09-13T14:57:26.842103+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":138,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561544+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:57:39.561544+0000","last_peered":"2024-09-13T14:57:39.561544+0000","last_clean":"2024-09-13T14:57:39.561544+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:57:39.561544+0000","last_undegraded":"2024-09-13T14:57:39.561544+0000","last_fullsized":"2024-09-13T14:57:39.561544+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"481'13818","reported_seq":13894,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:45.951465+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:57:45.951465+0000","last_peered":"2024-09-13T14:57:45.951465+0000","last_clean":"2024-09-13T14:57:45.951465+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:57:45.951465+0000","last_undegraded":"2024-09-13T14:57:45.951465+0000","last_fullsized":"2024-09-13T14:57:45.951465+0000","mapping_epoch":449,"log_start":"461'6300","ondisk_log_start":"461'6300","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145096704,"num_objects":2214,"num_object_clones":0,"num_object_copies":6642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13818,"num_write_kb":513024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":592,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851313+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:57:26.851313+0000","last_peered":"2024-09-13T14:57:26.851313+0000","last_clean":"2024-09-13T14:57:26.851313+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:57:26.851313+0000","last_undegraded":"2024-09-13T14:57:26.851313+0000","last_fullsized":"2024-09-13T14:57:26.851313+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":883,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.565215+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:57:39.565215+0000","last_peered":"2024-09-13T14:57:39.565215+0000","last_clean":"2024-09-13T14:57:39.565215+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:57:39.565215+0000","last_undegraded":"2024-09-13T14:57:39.565215+0000","last_fullsized":"2024-09-13T14:57:39.565215+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.849192+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:57:26.849192+0000","last_peered":"2024-09-13T14:57:26.849192+0000","last_clean":"2024-09-13T14:57:26.849192+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:57:26.849192+0000","last_undegraded":"2024-09-13T14:57:26.849192+0000","last_fullsized":"2024-09-13T14:57:26.849192+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"481'13596","reported_seq":13635,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.468528+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:57:48.468528+0000","last_peered":"2024-09-13T14:57:48.468528+0000","last_clean":"2024-09-13T14:57:48.468528+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:57:48.468528+0000","last_undegraded":"2024-09-13T14:57:48.468528+0000","last_fullsized":"2024-09-13T14:57:48.468528+0000","mapping_epoch":449,"log_start":"459'5000","ondisk_log_start":"459'5000","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13596,"num_write_kb":500416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":306,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.841724+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:57:26.841724+0000","last_peered":"2024-09-13T14:57:26.841724+0000","last_clean":"2024-09-13T14:57:26.841724+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:57:26.841724+0000","last_undegraded":"2024-09-13T14:57:26.841724+0000","last_fullsized":"2024-09-13T14:57:26.841724+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":482,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844008+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:57:26.844008+0000","last_peered":"2024-09-13T14:57:26.844008+0000","last_clean":"2024-09-13T14:57:26.844008+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:57:26.844008+0000","last_undegraded":"2024-09-13T14:57:26.844008+0000","last_fullsized":"2024-09-13T14:57:26.844008+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":193,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.847171+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:57:26.847171+0000","last_peered":"2024-09-13T14:57:26.847171+0000","last_clean":"2024-09-13T14:57:26.847171+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:57:26.847171+0000","last_undegraded":"2024-09-13T14:57:26.847171+0000","last_fullsized":"2024-09-13T14:57:26.847171+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":684,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553428+0000","last_change":"2024-09-13T14:56:00.671941+0000","last_active":"2024-09-13T14:57:39.553428+0000","last_peered":"2024-09-13T14:57:39.553428+0000","last_clean":"2024-09-13T14:57:39.553428+0000","last_became_active":"2024-09-13T14:56:00.668042+0000","last_became_peered":"2024-09-13T14:56:00.668042+0000","last_unstale":"2024-09-13T14:57:39.553428+0000","last_undegraded":"2024-09-13T14:57:39.553428+0000","last_fullsized":"2024-09-13T14:57:39.553428+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.375774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.375774+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:18.945468+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"481'13663","reported_seq":13703,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.948019+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:57:47.948019+0000","last_peered":"2024-09-13T14:57:47.948019+0000","last_clean":"2024-09-13T14:57:47.948019+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:57:47.948019+0000","last_undegraded":"2024-09-13T14:57:47.948019+0000","last_fullsized":"2024-09-13T14:57:47.948019+0000","mapping_epoch":449,"log_start":"462'6667","ondisk_log_start":"462'6667","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13663,"num_write_kb":504320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846315+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:57:26.846315+0000","last_peered":"2024-09-13T14:57:26.846315+0000","last_clean":"2024-09-13T14:57:26.846315+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:57:26.846315+0000","last_undegraded":"2024-09-13T14:57:26.846315+0000","last_fullsized":"2024-09-13T14:57:26.846315+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846836+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:57:26.846836+0000","last_peered":"2024-09-13T14:57:26.846836+0000","last_clean":"2024-09-13T14:57:26.846836+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:57:26.846836+0000","last_undegraded":"2024-09-13T14:57:26.846836+0000","last_fullsized":"2024-09-13T14:57:26.846836+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"481'13521","reported_seq":13561,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.159538+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:57:48.159538+0000","last_peered":"2024-09-13T14:57:48.159538+0000","last_clean":"2024-09-13T14:57:48.159538+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:57:48.159538+0000","last_undegraded":"2024-09-13T14:57:48.159538+0000","last_fullsized":"2024-09-13T14:57:48.159538+0000","mapping_epoch":449,"log_start":"467'7335","ondisk_log_start":"467'7335","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6186,"log_dups_size":0,"ondisk_log_size":6186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13521,"num_write_kb":498624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":299,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842025+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:57:26.842025+0000","last_peered":"2024-09-13T14:57:26.842025+0000","last_clean":"2024-09-13T14:57:26.842025+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:57:26.842025+0000","last_undegraded":"2024-09-13T14:57:26.842025+0000","last_fullsized":"2024-09-13T14:57:26.842025+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":853,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.563843+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:57:39.563843+0000","last_peered":"2024-09-13T14:57:39.563843+0000","last_clean":"2024-09-13T14:57:39.563843+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:57:39.563843+0000","last_undegraded":"2024-09-13T14:57:39.563843+0000","last_fullsized":"2024-09-13T14:57:39.563843+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":815,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.552619+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:57:39.552619+0000","last_peered":"2024-09-13T14:57:39.552619+0000","last_clean":"2024-09-13T14:57:39.552619+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:57:39.552619+0000","last_undegraded":"2024-09-13T14:57:39.552619+0000","last_fullsized":"2024-09-13T14:57:39.552619+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"481'13682","reported_seq":13722,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.970368+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:57:47.970368+0000","last_peered":"2024-09-13T14:57:47.970368+0000","last_clean":"2024-09-13T14:57:47.970368+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:57:47.970368+0000","last_undegraded":"2024-09-13T14:57:47.970368+0000","last_fullsized":"2024-09-13T14:57:47.970368+0000","mapping_epoch":449,"log_start":"462'6700","ondisk_log_start":"462'6700","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137232384,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13682,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":623,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.852717+0000","last_change":"2024-09-13T14:55:57.906683+0000","last_active":"2024-09-13T14:57:26.852717+0000","last_peered":"2024-09-13T14:57:26.852717+0000","last_clean":"2024-09-13T14:57:26.852717+0000","last_became_active":"2024-09-13T14:55:54.613915+0000","last_became_peered":"2024-09-13T14:55:54.613915+0000","last_unstale":"2024-09-13T14:57:26.852717+0000","last_undegraded":"2024-09-13T14:57:26.852717+0000","last_fullsized":"2024-09-13T14:57:26.852717+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.906630+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.906630+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:03.677753+0000","scrub_duration":0.001255616,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":1005,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842516+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:57:26.842516+0000","last_peered":"2024-09-13T14:57:26.842516+0000","last_clean":"2024-09-13T14:57:26.842516+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:57:26.842516+0000","last_undegraded":"2024-09-13T14:57:26.842516+0000","last_fullsized":"2024-09-13T14:57:26.842516+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":807,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.564247+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:57:39.564247+0000","last_peered":"2024-09-13T14:57:39.564247+0000","last_clean":"2024-09-13T14:57:39.564247+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:57:39.564247+0000","last_undegraded":"2024-09-13T14:57:39.564247+0000","last_fullsized":"2024-09-13T14:57:39.564247+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":447,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844227+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:57:26.844227+0000","last_peered":"2024-09-13T14:57:26.844227+0000","last_clean":"2024-09-13T14:57:26.844227+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:57:26.844227+0000","last_undegraded":"2024-09-13T14:57:26.844227+0000","last_fullsized":"2024-09-13T14:57:26.844227+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"481'13729","reported_seq":13804,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.616579+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:57:47.616579+0000","last_peered":"2024-09-13T14:57:47.616579+0000","last_clean":"2024-09-13T14:57:47.616579+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:57:47.616579+0000","last_undegraded":"2024-09-13T14:57:47.616579+0000","last_fullsized":"2024-09-13T14:57:47.616579+0000","mapping_epoch":449,"log_start":"461'6502","ondisk_log_start":"461'6502","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7227,"log_dups_size":0,"ondisk_log_size":7227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129564672,"num_objects":1977,"num_object_clones":0,"num_object_copies":5931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13729,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":682,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553615+0000","last_change":"2024-09-13T14:56:00.322316+0000","last_active":"2024-09-13T14:57:39.553615+0000","last_peered":"2024-09-13T14:57:39.553615+0000","last_clean":"2024-09-13T14:57:39.553615+0000","last_became_active":"2024-09-13T14:55:54.607589+0000","last_became_peered":"2024-09-13T14:55:54.607589+0000","last_unstale":"2024-09-13T14:57:39.553615+0000","last_undegraded":"2024-09-13T14:57:39.553615+0000","last_fullsized":"2024-09-13T14:57:39.553615+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.322230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.322230+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:10.503431+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":893,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.852670+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:57:26.852670+0000","last_peered":"2024-09-13T14:57:26.852670+0000","last_clean":"2024-09-13T14:57:26.852670+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:57:26.852670+0000","last_undegraded":"2024-09-13T14:57:26.852670+0000","last_fullsized":"2024-09-13T14:57:26.852670+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":138,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561151+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:57:39.561151+0000","last_peered":"2024-09-13T14:57:39.561151+0000","last_clean":"2024-09-13T14:57:39.561151+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:57:39.561151+0000","last_undegraded":"2024-09-13T14:57:39.561151+0000","last_fullsized":"2024-09-13T14:57:39.561151+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":138,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561350+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:57:39.561350+0000","last_peered":"2024-09-13T14:57:39.561350+0000","last_clean":"2024-09-13T14:57:39.561350+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:57:39.561350+0000","last_undegraded":"2024-09-13T14:57:39.561350+0000","last_fullsized":"2024-09-13T14:57:39.561350+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"481'13925","reported_seq":13965,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.478543+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:57:48.478543+0000","last_peered":"2024-09-13T14:57:48.478543+0000","last_clean":"2024-09-13T14:57:48.478543+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:57:48.478543+0000","last_undegraded":"2024-09-13T14:57:48.478543+0000","last_fullsized":"2024-09-13T14:57:48.478543+0000","mapping_epoch":449,"log_start":"459'5300","ondisk_log_start":"459'5300","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8625,"log_dups_size":0,"ondisk_log_size":8625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134545408,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13925,"num_write_kb":511296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":652,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851304+0000","last_change":"2024-09-13T14:55:58.908251+0000","last_active":"2024-09-13T14:57:26.851304+0000","last_peered":"2024-09-13T14:57:26.851304+0000","last_clean":"2024-09-13T14:57:26.851304+0000","last_became_active":"2024-09-13T14:55:54.619910+0000","last_became_peered":"2024-09-13T14:55:54.619910+0000","last_unstale":"2024-09-13T14:57:26.851304+0000","last_undegraded":"2024-09-13T14:57:26.851304+0000","last_fullsized":"2024-09-13T14:57:26.851304+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.908152+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.908152+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:01.217177+0000","scrub_duration":0.0048598549999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":833,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.564123+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:57:39.564123+0000","last_peered":"2024-09-13T14:57:39.564123+0000","last_clean":"2024-09-13T14:57:39.564123+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:57:39.564123+0000","last_undegraded":"2024-09-13T14:57:39.564123+0000","last_fullsized":"2024-09-13T14:57:39.564123+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":793,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553121+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:57:39.553121+0000","last_peered":"2024-09-13T14:57:39.553121+0000","last_clean":"2024-09-13T14:57:39.553121+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:57:39.553121+0000","last_undegraded":"2024-09-13T14:57:39.553121+0000","last_fullsized":"2024-09-13T14:57:39.553121+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842331+0000","last_change":"2024-09-13T14:53:08.771690+0000","last_active":"2024-09-13T14:57:26.842331+0000","last_peered":"2024-09-13T14:57:26.842331+0000","last_clean":"2024-09-13T14:57:26.842331+0000","last_became_active":"2024-09-13T14:53:08.771026+0000","last_became_peered":"2024-09-13T14:53:08.771026+0000","last_unstale":"2024-09-13T14:57:26.842331+0000","last_undegraded":"2024-09-13T14:57:26.842331+0000","last_fullsized":"2024-09-13T14:57:26.842331+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:55.877854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"481'13731","reported_seq":13771,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.953427+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:57:47.953427+0000","last_peered":"2024-09-13T14:57:47.953427+0000","last_clean":"2024-09-13T14:57:47.953427+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:57:47.953427+0000","last_undegraded":"2024-09-13T14:57:47.953427+0000","last_fullsized":"2024-09-13T14:57:47.953427+0000","mapping_epoch":449,"log_start":"462'6700","ondisk_log_start":"462'6700","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128647168,"num_objects":1963,"num_object_clones":0,"num_object_copies":5889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13731,"num_write_kb":502208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":339,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842073+0000","last_change":"2024-09-13T14:56:00.644560+0000","last_active":"2024-09-13T14:57:26.842073+0000","last_peered":"2024-09-13T14:57:26.842073+0000","last_clean":"2024-09-13T14:57:26.842073+0000","last_became_active":"2024-09-13T14:55:54.621234+0000","last_became_peered":"2024-09-13T14:55:54.621234+0000","last_unstale":"2024-09-13T14:57:26.842073+0000","last_undegraded":"2024-09-13T14:57:26.842073+0000","last_fullsized":"2024-09-13T14:57:26.842073+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.644466+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.644466+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.375763+0000","scrub_duration":0.0093267220000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1096,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842157+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:57:26.842157+0000","last_peered":"2024-09-13T14:57:26.842157+0000","last_clean":"2024-09-13T14:57:26.842157+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:57:26.842157+0000","last_undegraded":"2024-09-13T14:57:26.842157+0000","last_fullsized":"2024-09-13T14:57:26.842157+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":199,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846679+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:57:26.846679+0000","last_peered":"2024-09-13T14:57:26.846679+0000","last_clean":"2024-09-13T14:57:26.846679+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:57:26.846679+0000","last_undegraded":"2024-09-13T14:57:26.846679+0000","last_fullsized":"2024-09-13T14:57:26.846679+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":814,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.551722+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:57:39.551722+0000","last_peered":"2024-09-13T14:57:39.551722+0000","last_clean":"2024-09-13T14:57:39.551722+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:57:39.551722+0000","last_undegraded":"2024-09-13T14:57:39.551722+0000","last_fullsized":"2024-09-13T14:57:39.551722+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"481'13863","reported_seq":13903,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.311058+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:57:48.311058+0000","last_peered":"2024-09-13T14:57:48.311058+0000","last_clean":"2024-09-13T14:57:48.311058+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:57:48.311058+0000","last_undegraded":"2024-09-13T14:57:48.311058+0000","last_fullsized":"2024-09-13T14:57:48.311058+0000","mapping_epoch":449,"log_start":"461'6642","ondisk_log_start":"461'6642","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7221,"log_dups_size":0,"ondisk_log_size":7221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13863,"num_write_kb":510720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":500,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844258+0000","last_change":"2024-09-13T14:55:59.724453+0000","last_active":"2024-09-13T14:57:26.844258+0000","last_peered":"2024-09-13T14:57:26.844258+0000","last_clean":"2024-09-13T14:57:26.844258+0000","last_became_active":"2024-09-13T14:55:54.613965+0000","last_became_peered":"2024-09-13T14:55:54.613965+0000","last_unstale":"2024-09-13T14:57:26.844258+0000","last_undegraded":"2024-09-13T14:57:26.844258+0000","last_fullsized":"2024-09-13T14:57:26.844258+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.723985+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.723985+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:11.061534+0000","scrub_duration":0.0085871660000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":426,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842116+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:57:26.842116+0000","last_peered":"2024-09-13T14:57:26.842116+0000","last_clean":"2024-09-13T14:57:26.842116+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:57:26.842116+0000","last_undegraded":"2024-09-13T14:57:26.842116+0000","last_fullsized":"2024-09-13T14:57:26.842116+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1477,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553112+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:57:39.553112+0000","last_peered":"2024-09-13T14:57:39.553112+0000","last_clean":"2024-09-13T14:57:39.553112+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:57:39.553112+0000","last_undegraded":"2024-09-13T14:57:39.553112+0000","last_fullsized":"2024-09-13T14:57:39.553112+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":795,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.554065+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:57:39.554065+0000","last_peered":"2024-09-13T14:57:39.554065+0000","last_clean":"2024-09-13T14:57:39.554065+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:57:39.554065+0000","last_undegraded":"2024-09-13T14:57:39.554065+0000","last_fullsized":"2024-09-13T14:57:39.554065+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"481'13546","reported_seq":13587,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.934538+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:57:47.934538+0000","last_peered":"2024-09-13T14:57:47.934538+0000","last_clean":"2024-09-13T14:57:47.934538+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:57:47.934538+0000","last_undegraded":"2024-09-13T14:57:47.934538+0000","last_fullsized":"2024-09-13T14:57:47.934538+0000","mapping_epoch":449,"log_start":"461'6483","ondisk_log_start":"461'6483","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131137560,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13546,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":520,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844176+0000","last_change":"2024-09-13T14:56:00.740088+0000","last_active":"2024-09-13T14:57:26.844176+0000","last_peered":"2024-09-13T14:57:26.844176+0000","last_clean":"2024-09-13T14:57:26.844176+0000","last_became_active":"2024-09-13T14:55:54.621088+0000","last_became_peered":"2024-09-13T14:55:54.621088+0000","last_unstale":"2024-09-13T14:57:26.844176+0000","last_undegraded":"2024-09-13T14:57:26.844176+0000","last_fullsized":"2024-09-13T14:57:26.844176+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.739931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.739931+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.959219+0000","scrub_duration":0.0093782959999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":804,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.552774+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:57:39.552774+0000","last_peered":"2024-09-13T14:57:39.552774+0000","last_clean":"2024-09-13T14:57:39.552774+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:57:39.552774+0000","last_undegraded":"2024-09-13T14:57:39.552774+0000","last_fullsized":"2024-09-13T14:57:39.552774+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":897,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851929+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:57:26.851929+0000","last_peered":"2024-09-13T14:57:26.851929+0000","last_clean":"2024-09-13T14:57:26.851929+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:57:26.851929+0000","last_undegraded":"2024-09-13T14:57:26.851929+0000","last_fullsized":"2024-09-13T14:57:26.851929+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":138,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561256+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:57:39.561256+0000","last_peered":"2024-09-13T14:57:39.561256+0000","last_clean":"2024-09-13T14:57:39.561256+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:57:39.561256+0000","last_undegraded":"2024-09-13T14:57:39.561256+0000","last_fullsized":"2024-09-13T14:57:39.561256+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"481'13877","reported_seq":13917,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:48.129975+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:57:48.129975+0000","last_peered":"2024-09-13T14:57:48.129975+0000","last_clean":"2024-09-13T14:57:48.129975+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:57:48.129975+0000","last_undegraded":"2024-09-13T14:57:48.129975+0000","last_fullsized":"2024-09-13T14:57:48.129975+0000","mapping_epoch":449,"log_start":"467'7656","ondisk_log_start":"467'7656","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6221,"log_dups_size":0,"ondisk_log_size":6221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135856128,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13877,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":199,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846943+0000","last_change":"2024-09-13T14:56:00.807834+0000","last_active":"2024-09-13T14:57:26.846943+0000","last_peered":"2024-09-13T14:57:26.846943+0000","last_clean":"2024-09-13T14:57:26.846943+0000","last_became_active":"2024-09-13T14:55:54.609192+0000","last_became_peered":"2024-09-13T14:55:54.609192+0000","last_unstale":"2024-09-13T14:57:26.846943+0000","last_undegraded":"2024-09-13T14:57:26.846943+0000","last_fullsized":"2024-09-13T14:57:26.846943+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.807561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.807561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:44.901235+0000","scrub_duration":0.012160298999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":861,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.567118+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:57:39.567118+0000","last_peered":"2024-09-13T14:57:39.567118+0000","last_clean":"2024-09-13T14:57:39.567118+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:57:39.567118+0000","last_undegraded":"2024-09-13T14:57:39.567118+0000","last_fullsized":"2024-09-13T14:57:39.567118+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1179,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.552273+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:57:39.552273+0000","last_peered":"2024-09-13T14:57:39.552273+0000","last_clean":"2024-09-13T14:57:39.552273+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:57:39.552273+0000","last_undegraded":"2024-09-13T14:57:39.552273+0000","last_fullsized":"2024-09-13T14:57:39.552273+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":851,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.563669+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:57:39.563669+0000","last_peered":"2024-09-13T14:57:39.563669+0000","last_clean":"2024-09-13T14:57:39.563669+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:57:39.563669+0000","last_undegraded":"2024-09-13T14:57:39.563669+0000","last_fullsized":"2024-09-13T14:57:39.563669+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"481'13855","reported_seq":13895,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:47.931211+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:57:47.931211+0000","last_peered":"2024-09-13T14:57:47.931211+0000","last_clean":"2024-09-13T14:57:47.931211+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:57:47.931211+0000","last_undegraded":"2024-09-13T14:57:47.931211+0000","last_fullsized":"2024-09-13T14:57:47.931211+0000","mapping_epoch":449,"log_start":"462'6784","ondisk_log_start":"462'6784","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139395072,"num_objects":2127,"num_object_clones":0,"num_object_copies":6381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13855,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":712,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.565755+0000","last_change":"2024-09-13T14:56:00.000186+0000","last_active":"2024-09-13T14:57:39.565755+0000","last_peered":"2024-09-13T14:57:39.565755+0000","last_clean":"2024-09-13T14:57:39.565755+0000","last_became_active":"2024-09-13T14:55:54.611253+0000","last_became_peered":"2024-09-13T14:55:54.611253+0000","last_unstale":"2024-09-13T14:57:39.565755+0000","last_undegraded":"2024-09-13T14:57:39.565755+0000","last_fullsized":"2024-09-13T14:57:39.565755+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.999712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.999712+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:46.770712+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":142,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561448+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:57:39.561448+0000","last_peered":"2024-09-13T14:57:39.561448+0000","last_clean":"2024-09-13T14:57:39.561448+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:57:39.561448+0000","last_undegraded":"2024-09-13T14:57:39.561448+0000","last_fullsized":"2024-09-13T14:57:39.561448+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":899,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844399+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:57:26.844399+0000","last_peered":"2024-09-13T14:57:26.844399+0000","last_clean":"2024-09-13T14:57:26.844399+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:57:26.844399+0000","last_undegraded":"2024-09-13T14:57:26.844399+0000","last_fullsized":"2024-09-13T14:57:26.844399+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":193,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846453+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:57:26.846453+0000","last_peered":"2024-09-13T14:57:26.846453+0000","last_clean":"2024-09-13T14:57:26.846453+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:57:26.846453+0000","last_undegraded":"2024-09-13T14:57:26.846453+0000","last_fullsized":"2024-09-13T14:57:26.846453+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846752+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:57:26.846752+0000","last_peered":"2024-09-13T14:57:26.846752+0000","last_clean":"2024-09-13T14:57:26.846752+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:57:26.846752+0000","last_undegraded":"2024-09-13T14:57:26.846752+0000","last_fullsized":"2024-09-13T14:57:26.846752+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":189,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.849118+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:57:26.849118+0000","last_peered":"2024-09-13T14:57:26.849118+0000","last_clean":"2024-09-13T14:57:26.849118+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:57:26.849118+0000","last_undegraded":"2024-09-13T14:57:26.849118+0000","last_fullsized":"2024-09-13T14:57:26.849118+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":790,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.553798+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:57:39.553798+0000","last_peered":"2024-09-13T14:57:39.553798+0000","last_clean":"2024-09-13T14:57:39.553798+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:57:39.553798+0000","last_undegraded":"2024-09-13T14:57:39.553798+0000","last_fullsized":"2024-09-13T14:57:39.553798+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":480,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851747+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:57:26.851747+0000","last_peered":"2024-09-13T14:57:26.851747+0000","last_clean":"2024-09-13T14:57:26.851747+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:57:26.851747+0000","last_undegraded":"2024-09-13T14:57:26.851747+0000","last_fullsized":"2024-09-13T14:57:26.851747+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":294,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.851510+0000","last_change":"2024-09-13T14:56:06.913381+0000","last_active":"2024-09-13T14:57:26.851510+0000","last_peered":"2024-09-13T14:57:26.851510+0000","last_clean":"2024-09-13T14:57:26.851510+0000","last_became_active":"2024-09-13T14:56:00.679102+0000","last_became_peered":"2024-09-13T14:56:00.679102+0000","last_unstale":"2024-09-13T14:57:26.851510+0000","last_undegraded":"2024-09-13T14:57:26.851510+0000","last_fullsized":"2024-09-13T14:57:26.851510+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:06.913154+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:06.913154+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:14.474904+0000","scrub_duration":0.0026651819999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":446,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842633+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:57:26.842633+0000","last_peered":"2024-09-13T14:57:26.842633+0000","last_clean":"2024-09-13T14:57:26.842633+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:57:26.842633+0000","last_undegraded":"2024-09-13T14:57:26.842633+0000","last_fullsized":"2024-09-13T14:57:26.842633+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":795,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.563956+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:57:39.563956+0000","last_peered":"2024-09-13T14:57:39.563956+0000","last_clean":"2024-09-13T14:57:39.563956+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:57:39.563956+0000","last_undegraded":"2024-09-13T14:57:39.563956+0000","last_fullsized":"2024-09-13T14:57:39.563956+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561821+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:57:39.561821+0000","last_peered":"2024-09-13T14:57:39.561821+0000","last_clean":"2024-09-13T14:57:39.561821+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:57:39.561821+0000","last_undegraded":"2024-09-13T14:57:39.561821+0000","last_fullsized":"2024-09-13T14:57:39.561821+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":851,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.563577+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:57:39.563577+0000","last_peered":"2024-09-13T14:57:39.563577+0000","last_clean":"2024-09-13T14:57:39.563577+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:57:39.563577+0000","last_undegraded":"2024-09-13T14:57:39.563577+0000","last_fullsized":"2024-09-13T14:57:39.563577+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":478,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.844327+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:57:26.844327+0000","last_peered":"2024-09-13T14:57:26.844327+0000","last_clean":"2024-09-13T14:57:26.844327+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:57:26.844327+0000","last_undegraded":"2024-09-13T14:57:26.844327+0000","last_fullsized":"2024-09-13T14:57:26.844327+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":882,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.563751+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:57:39.563751+0000","last_peered":"2024-09-13T14:57:39.563751+0000","last_clean":"2024-09-13T14:57:39.563751+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:57:39.563751+0000","last_undegraded":"2024-09-13T14:57:39.563751+0000","last_fullsized":"2024-09-13T14:57:39.563751+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":138,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561639+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:57:39.561639+0000","last_peered":"2024-09-13T14:57:39.561639+0000","last_clean":"2024-09-13T14:57:39.561639+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:57:39.561639+0000","last_undegraded":"2024-09-13T14:57:39.561639+0000","last_fullsized":"2024-09-13T14:57:39.561639+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":815,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.551737+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:57:39.551737+0000","last_peered":"2024-09-13T14:57:39.551737+0000","last_clean":"2024-09-13T14:57:39.551737+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:57:39.551737+0000","last_undegraded":"2024-09-13T14:57:39.551737+0000","last_fullsized":"2024-09-13T14:57:39.551737+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":864,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.567063+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:57:39.567063+0000","last_peered":"2024-09-13T14:57:39.567063+0000","last_clean":"2024-09-13T14:57:39.567063+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:57:39.567063+0000","last_undegraded":"2024-09-13T14:57:39.567063+0000","last_fullsized":"2024-09-13T14:57:39.567063+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":178,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.848745+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:57:26.848745+0000","last_peered":"2024-09-13T14:57:26.848745+0000","last_clean":"2024-09-13T14:57:26.848745+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:57:26.848745+0000","last_undegraded":"2024-09-13T14:57:26.848745+0000","last_fullsized":"2024-09-13T14:57:26.848745+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":761,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.552961+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:57:39.552961+0000","last_peered":"2024-09-13T14:57:39.552961+0000","last_clean":"2024-09-13T14:57:39.552961+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:57:39.552961+0000","last_undegraded":"2024-09-13T14:57:39.552961+0000","last_fullsized":"2024-09-13T14:57:39.552961+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":137,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561346+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:57:39.561346+0000","last_peered":"2024-09-13T14:57:39.561346+0000","last_clean":"2024-09-13T14:57:39.561346+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:57:39.561346+0000","last_undegraded":"2024-09-13T14:57:39.561346+0000","last_fullsized":"2024-09-13T14:57:39.561346+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846775+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:57:26.846775+0000","last_peered":"2024-09-13T14:57:26.846775+0000","last_clean":"2024-09-13T14:57:26.846775+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:57:26.846775+0000","last_undegraded":"2024-09-13T14:57:26.846775+0000","last_fullsized":"2024-09-13T14:57:26.846775+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":446,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842189+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:57:26.842189+0000","last_peered":"2024-09-13T14:57:26.842189+0000","last_clean":"2024-09-13T14:57:26.842189+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:57:26.842189+0000","last_undegraded":"2024-09-13T14:57:26.842189+0000","last_fullsized":"2024-09-13T14:57:26.842189+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.848732+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:57:26.848732+0000","last_peered":"2024-09-13T14:57:26.848732+0000","last_clean":"2024-09-13T14:57:26.848732+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:57:26.848732+0000","last_undegraded":"2024-09-13T14:57:26.848732+0000","last_fullsized":"2024-09-13T14:57:26.848732+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":412,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.842420+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:57:26.842420+0000","last_peered":"2024-09-13T14:57:26.842420+0000","last_clean":"2024-09-13T14:57:26.842420+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:57:26.842420+0000","last_undegraded":"2024-09-13T14:57:26.842420+0000","last_fullsized":"2024-09-13T14:57:26.842420+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":874,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.564153+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:57:39.564153+0000","last_peered":"2024-09-13T14:57:39.564153+0000","last_clean":"2024-09-13T14:57:39.564153+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:57:39.564153+0000","last_undegraded":"2024-09-13T14:57:39.564153+0000","last_fullsized":"2024-09-13T14:57:39.564153+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":848,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.564760+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:57:39.564760+0000","last_peered":"2024-09-13T14:57:39.564760+0000","last_clean":"2024-09-13T14:57:39.564760+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:57:39.564760+0000","last_undegraded":"2024-09-13T14:57:39.564760+0000","last_fullsized":"2024-09-13T14:57:39.564760+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":480,"state":"active+clean","last_fresh":"2024-09-13T14:57:26.846246+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:57:26.846246+0000","last_peered":"2024-09-13T14:57:26.846246+0000","last_clean":"2024-09-13T14:57:26.846246+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:57:26.846246+0000","last_undegraded":"2024-09-13T14:57:26.846246+0000","last_fullsized":"2024-09-13T14:57:26.846246+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":144,"reported_epoch":481,"state":"active+clean","last_fresh":"2024-09-13T14:57:39.561718+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:57:39.561718+0000","last_peered":"2024-09-13T14:57:39.561718+0000","last_clean":"2024-09-13T14:57:39.561718+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:57:39.561718+0000","last_undegraded":"2024-09-13T14:57:39.561718+0000","last_fullsized":"2024-09-13T14:57:39.561718+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2158362648,"num_objects":32935,"num_object_clones":0,"num_object_copies":98805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32935,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":219669,"num_write_kb":8083265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6517305344,"data_stored":6517293128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116835,"ondisk_log_size":116835,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398060,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446080,"kb_used_data":545912,"kb_used_omap":1,"kb_used_meta":900158,"kb_avail":92299072,"statfs":{"total":95995035648,"available":94514249728,"internally_reserved":0,"allocated":559013888,"data_stored":556483696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921762136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594288,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818676,"kb_used_data":924204,"kb_used_omap":1,"kb_used_meta":894462,"kb_avail":91926476,"statfs":{"total":95995035648,"available":94132711424,"internally_reserved":0,"allocated":946384896,"data_stored":943866314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915929432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665742,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579116,"kb_used_data":648932,"kb_used_omap":9,"kb_used_meta":930166,"kb_avail":92166036,"statfs":{"total":95995035648,"available":94378020864,"internally_reserved":0,"allocated":664506368,"data_stored":661992261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":952490184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986744,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1995136,"kb_used_data":1086904,"kb_used_omap":1,"kb_used_meta":908158,"kb_avail":91750016,"statfs":{"total":95995035648,"available":93952016384,"internally_reserved":0,"allocated":1112989696,"data_stored":1110459458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":929954136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697304,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1792940,"kb_used_data":663204,"kb_used_omap":4,"kb_used_meta":1129723,"kb_avail":91952212,"statfs":{"total":95995035648,"available":94159065088,"internally_reserved":0,"allocated":679120896,"data_stored":676606870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1156836427},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860830,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717936,"kb_used_data":851432,"kb_used_omap":8,"kb_used_meta":866487,"kb_avail":92027216,"statfs":{"total":95995035648,"available":94235869184,"internally_reserved":0,"allocated":871866368,"data_stored":869348760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":887283532},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020501,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1088744,"kb_used_data":441120,"kb_used_omap":9,"kb_used_meta":647606,"kb_avail":92656408,"statfs":{"total":95995035648,"available":94880161792,"internally_reserved":0,"allocated":451706880,"data_stored":449196327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":663149381},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187892,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2341804,"kb_used_data":1352484,"kb_used_omap":7,"kb_used_meta":989304,"kb_avail":91403348,"statfs":{"total":95995035648,"available":93597028352,"internally_reserved":0,"allocated":1384943616,"data_stored":1382429257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":1013047627},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1354108928,"data_stored":1354104856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":435552256,"data_stored":435552256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":861995008,"data_stored":861995008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":655097856,"data_stored":655097856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1081151488,"data_stored":1081147416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":656146432,"data_stored":656146432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":939134976,"data_stored":939130904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":534118400,"data_stored":534118400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:57:51.605 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:57:51.605 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:57:51.605 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T14:57:51.605 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.353455 2024-09-13T14:57:51.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 0.3534551230325673 2024-09-13T14:57:51.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:51.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:51.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:51 smithi017 ceph-mon[103001]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:51.812 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:51 smithi017 ceph-mon[107556]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:51.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:51.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:52.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:51 smithi189 ceph-mon[90992]: from='client.37293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:52.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:52.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:52.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:52.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:52.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:52.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:52.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:52.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:52.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:52.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:52.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:52 smithi017 ceph-mon[103001]: pgmap v1592: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-13T14:57:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:52 smithi017 ceph-mon[107556]: pgmap v1592: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-13T14:57:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:52 smithi189 ceph-mon[90992]: pgmap v1592: 109 pgs: 109 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-13T14:57:53.013 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:53.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:53.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:53.456 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:53.493 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:53.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:53.628 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:53.730 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:53.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.770 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:53.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:53.871 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:53.895 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:53.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:53.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:54.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:54.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:54.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:54.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:54.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:54.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:54.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:54.626 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:57:54.689 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[103001]: pgmap v1593: 109 pgs: 109 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:57:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3653256198' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[107556]: pgmap v1593: 109 pgs: 109 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:57:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3653256198' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:54 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:54.733 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:54.835 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.876 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:54.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:54 smithi189 ceph-mon[90992]: pgmap v1593: 109 pgs: 109 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:57:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3653256198' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:54 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:55.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:55.117 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.153 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:55.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.254 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.277 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:55.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:55.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:55.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:55.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:55.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:55.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:55.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:55.921 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[103001]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:57:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:55 smithi017 ceph-mon[107556]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:57:55.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:55 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:55 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:56.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:55 smithi189 ceph-mon[90992]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:57:56.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:56.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:56.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:56.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:56.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:56.806 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:56.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:56 smithi017 ceph-mon[107556]: pgmap v1595: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:56 smithi017 ceph-mon[103001]: pgmap v1595: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:56.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:56 smithi189 ceph-mon[90992]: pgmap v1595: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:57.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:57.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:57.313 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:57.356 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:57:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:57:57] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.51.0" 2024-09-13T14:57:57.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:57:57.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:57:57.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:57.433 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:57.456 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:57:57.456 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:57.479 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:57:57.479 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:57.508 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:57:57.509 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:57.518 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.547 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:57:57.547 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:57.548 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:57.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:57.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:57:57.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:57.627 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:57:57.627 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:57.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:57.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:57.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/380074398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/380074398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:57.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:58.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/380074398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:58.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.128 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:58.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.229 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.253 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.355 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.383 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.485 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.514 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:58.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.743 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:58.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:58.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:58.981 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[103001]: pgmap v1596: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:58.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:58.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:58.982 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[107556]: pgmap v1596: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:58.982 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:58.982 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:58 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:59.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:59.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:59.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:59.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:59.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:59.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:58 smithi189 ceph-mon[90992]: pgmap v1596: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:57:59.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:59.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:58 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:57:59.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:59.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:59.460 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.502 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:59.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:59.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:59.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:57:59.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:59.881 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.915 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:57:59.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:00.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:00.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:00.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:58:00.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:58:00.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:00.122 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:58:00.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:58:00.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:57:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:00.156 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:58:00.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:58:00.157 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:57:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:00.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.206 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:00.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:00.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:00.440 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.475 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:00.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:00.576 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:00.621 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:00.723 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.758 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:00.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:00.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:58:00.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:00.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:00.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:00.923 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:00 smithi189 ceph-mon[90992]: pgmap v1597: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:58:01.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.041 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:01.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:01.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:00 smithi017 ceph-mon[103001]: pgmap v1597: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:58:01.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:00 smithi017 ceph-mon[107556]: pgmap v1597: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:58:01.138 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:01.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:01.184 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:01.184 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:01.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:01.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:01.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:01.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:01.496 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:01.598 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.635 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:01.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.737 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.779 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:01.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:01.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:01.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:58:01.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:02.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:02.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:02.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:02.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:02.325 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.358 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.501 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:02.603 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:02.630 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:02.632 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:02.734 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.762 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.864 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:02.896 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:02.998 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.023 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.148 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:02 smithi017 ceph-mon[103001]: pgmap v1598: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:58:03.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:02 smithi017 ceph-mon[107556]: pgmap v1598: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:58:03.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:02 smithi189 ceph-mon[90992]: pgmap v1598: 109 pgs: 109 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:58:03.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:03.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:03.534 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:03.572 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:03.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:03.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:03.823 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:03.864 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:03.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.106 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.134 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:04.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:04.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:04.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:04.370 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:04.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:04.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:04.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:04.646 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:04.686 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:04.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:04.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:04.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:04.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:04.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:05.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:04 smithi017 ceph-mon[103001]: pgmap v1599: 109 pgs: 109 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:58:05.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:04 smithi017 ceph-mon[107556]: pgmap v1599: 109 pgs: 109 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:58:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:05.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:05.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.252 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:05.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:05.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:04 smithi189 ceph-mon[90992]: pgmap v1599: 109 pgs: 109 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-09-13T14:58:05.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:05.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:05.493 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.527 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:05.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:05.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.735 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:05.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:05.778 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:05.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.086 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.188 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.226 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:06.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:06.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:06.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:06.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:06.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:06.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:06.614 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.644 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:06.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:06.816 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 0.353455 (8231642) 2024-09-13T14:58:06.889 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:06.922 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:07.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:07.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:07.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:07.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:07.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:06 smithi017 ceph-mon[103001]: pgmap v1600: 109 pgs: 109 active+clean; 1.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:58:07.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673} : dispatch 2024-09-13T14:58:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:06 smithi017 ceph-mon[107556]: pgmap v1600: 109 pgs: 109 active+clean; 1.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:58:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673} : dispatch 2024-09-13T14:58:07.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:06 smithi189 ceph-mon[90992]: pgmap v1600: 109 pgs: 109 active+clean; 1.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-13T14:58:07.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673} : dispatch 2024-09-13T14:58:07.296 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.326 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:07.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.428 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.458 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:07.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.702 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:07] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.51.0" 2024-09-13T14:58:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:07.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:07.817 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:58:07.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.870 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:07.972 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:08.009 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:08.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:08.110 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:08.149 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673}]': finished 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[103001]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:58:08.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673}]': finished 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:08.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:07 smithi017 ceph-mon[107556]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:58:08.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/171533874' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.3534551230325673}]': finished 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1553902923' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:08.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:07 smithi189 ceph-mon[90992]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:58:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.293 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:08.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:08.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:08.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:08.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:08.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:08.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:08.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:08.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:08.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:08.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:08.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:09.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:09.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:08 smithi017 ceph-mon[103001]: pgmap v1602: 109 pgs: 109 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-13T14:58:09.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:08 smithi017 ceph-mon[107556]: pgmap v1602: 109 pgs: 109 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-13T14:58:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:09.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:08 smithi189 ceph-mon[90992]: pgmap v1602: 109 pgs: 109 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-13T14:58:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:09.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:09.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:09.659 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:09.689 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:09.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:09.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:09.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:09.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:10.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.109 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:10.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:09 smithi017 ceph-mon[103001]: 1.14 scrub starts 2024-09-13T14:58:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:09 smithi017 ceph-mon[103001]: 1.14 scrub ok 2024-09-13T14:58:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:09 smithi017 ceph-mon[107556]: 1.14 scrub starts 2024-09-13T14:58:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:09 smithi017 ceph-mon[107556]: 1.14 scrub ok 2024-09-13T14:58:10.210 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:09 smithi189 ceph-mon[90992]: 1.14 scrub starts 2024-09-13T14:58:10.214 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:09 smithi189 ceph-mon[90992]: 1.14 scrub ok 2024-09-13T14:58:10.247 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:10.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:10.488 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:10.520 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:10.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:10.757 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.787 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:10.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:10.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:10 smithi017 ceph-mon[103001]: pgmap v1604: 109 pgs: 109 active+clean; 859 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:58:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:10 smithi017 ceph-mon[107556]: pgmap v1604: 109 pgs: 109 active+clean; 859 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:58:11.027 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:11.056 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:11.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:11.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:11.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:58:11.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:11.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:10 smithi189 ceph-mon[90992]: pgmap v1604: 109 pgs: 109 active+clean; 859 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T14:58:11.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:11.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:11.435 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:11.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:11.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:11.698 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:11.728 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:11.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:11.974 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.173 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.213 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:12.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:12.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:12.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:12.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:12.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:12.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:12.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:12.749 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.776 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:58:12.831 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:12.865 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:58:12.866 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:12.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:12.899 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:58:12.899 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:12.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:12.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:58:12.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:12.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:58:12.972 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:13.002 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:58:13.002 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[103001]: pgmap v1605: 109 pgs: 109 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3333671376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[107556]: pgmap v1605: 109 pgs: 109 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:13.003 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:12 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3333671376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:13.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:13.042 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:58:13.042 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:13.075 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:58:13.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:13.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:13.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:13.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:13.222 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:13.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:13.223 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:58:13.223 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:13.223 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:58:13.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:58:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:12 smithi189 ceph-mon[90992]: pgmap v1605: 109 pgs: 109 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2024-09-13T14:58:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:13.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:12 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3333671376' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:13.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:13.380 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88010 9.c 9:36a45f6d:::benchmark_data_smithi017_339564_object88009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.99609830399999, 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23807841599999999, 2024-09-13T14:58:13.381 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88010 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377061+0000", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377063+0000", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8300000000000001e-06 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.377118+0000", 2024-09-13T14:58:13.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4843999999999999e-05 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380264+0000", 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003145994 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380275+0000", 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1649e-05 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054572799999999999 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381007+0000", 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018597899999999999 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.381155+0000", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014763800000000001 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387265+0000", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061102819999999999 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614900+0000", 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227635122 2024-09-13T14:58:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615054+0000", 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015373700000000001 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615094+0000", 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0738999999999997e-05 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615139+0000", 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4873999999999998e-05 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88013 9.c 9:3b76033e:::benchmark_data_smithi017_339564_object88012:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.99233829799999, 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23455462899999999, 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88013 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380821+0000", 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380822+0000", 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3289999999999999e-06 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380883+0000", 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0396000000000002e-05 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380895+0000", 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2823000000000001e-05 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380903+0000", 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8860000000000004e-06 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384158+0000", 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003254346 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384318+0000", 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016008400000000001 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384451+0000", 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013359999999999999 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614962+0000", 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.230510786 2024-09-13T14:58:13.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615189+0000", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022717899999999999 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615283+0000", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3936000000000003e-05 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615340+0000", 2024-09-13T14:58:13.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6261999999999998e-05 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615376+0000", 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6001999999999999e-05 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.395 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88014 9.c 9:30066bc1:::benchmark_data_smithi017_339564_object88013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.99224241899998, 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23458493899999999, 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88014 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380917+0000", 2024-09-13T14:58:13.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380969+0000", 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1617999999999999e-05 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380978+0000", 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4979999999999995e-06 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.380985+0000", 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5610000000000004e-06 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387301+0000", 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063155529999999998 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387451+0000", 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150414 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.387614+0000", 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016278399999999999 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615015+0000", 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.227400877 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615195+0000", 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017990500000000001 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615419+0000", 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224207 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615466+0000", 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7579e-05 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615502+0000", 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5425e-05 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:88018 9.4 e381/376 9:2920a091:::benchmark_data_smithi017_339564_object88017:head v 381'5555, mlcod=381'5553)", 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:58:13.402 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.99043249499999, 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23245458799999999, 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382727+0000", 2024-09-13T14:58:13.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382728+0000", 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6099999999999999e-07 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382749+0000", 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1486000000000001e-05 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382760+0000", 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1107999999999999e-05 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382767+0000", 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8820000000000003e-06 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384135+0000", 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013674410000000001 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.384156+0000", 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1027999999999999e-05 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615151+0000", 2024-09-13T14:58:13.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23099550899999999 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615181+0000", 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0173000000000001e-05 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88020 9.c 9:32f382c7:::benchmark_data_smithi017_339564_object88019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.407 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.990206822, 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23295475700000001, 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88020 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382952+0000", 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.382954+0000", 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1620000000000001e-06 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383001+0000", 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7929000000000001e-05 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383013+0000", 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1324e-05 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.383020+0000", 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5340000000000002e-06 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390514+0000", 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074939789999999996 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390651+0000", 2024-09-13T14:58:13.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013669299999999999 2024-09-13T14:58:13.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390787+0000", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135773 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615200+0000", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22441285599999999 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615234+0000", 2024-09-13T14:58:13.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4634000000000001e-05 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615794+0000", 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055964299999999997 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615859+0000", 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5318e-05 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615907+0000", 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7911999999999998e-05 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88025 9.8 9:1a0cff2e:::benchmark_data_smithi017_339564_object88024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.98245458000002, 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23531686499999999, 2024-09-13T14:58:13.415 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88025 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:58:13.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390705+0000", 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390706+0000", 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3009999999999999e-06 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390728+0000", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1959000000000001e-05 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390741+0000", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2892e-05 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390748+0000", 2024-09-13T14:58:13.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390808+0000", 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0133000000000002e-05 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.390954+0000", 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014599599999999999 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.391077+0000", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012244 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621755+0000", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23067821399999999 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621803+0000", 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8112999999999998e-05 2024-09-13T14:58:13.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625781+0000", 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039782510000000004 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625827+0000", 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5982000000000001e-05 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626021+0000", 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019408899999999999 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:88029 9.8 9:1879118d:::benchmark_data_smithi017_339564_object88028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 429.98014365300003, 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.233198557, 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.422 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88029 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393016+0000", 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393017+0000", 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.37e-06 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393040+0000", 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2705e-05 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393052+0000", 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1895e-05 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.393061+0000", 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1209999999999996e-06 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.614926+0000", 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.221865756 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615166+0000", 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000239127 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.615958+0000", 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079216400000000004 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.621846+0000", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058882079999999998 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.625766+0000", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039203490000000001 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626034+0000", 2024-09-13T14:58:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026819300000000001 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626171+0000", 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136966 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:03.626214+0000", 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2703000000000001e-05 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.59229632900002, 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30486941899999997, 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.429 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780863+0000", 2024-09-13T14:58:13.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780865+0000", 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.629e-06 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780911+0000", 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6511999999999998e-05 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780922+0000", 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1454000000000001e-05 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.780930+0000", 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9019999999999999e-06 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781098+0000", 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016806699999999999 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781253+0000", 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015428100000000001 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:13.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.781490+0000", 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023734 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789140+0000", 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076497320000000002 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800223+0000", 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011082834999999999 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085715+0000", 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28549267499999997 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085732+0000", 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6991999999999999e-05 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.2 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:58:13.435 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.58775810600002, 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30060562000000002, 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.436 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785401+0000", 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785402+0000", 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8100000000000003e-07 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785421+0000", 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9623000000000002e-05 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785429+0000", 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.785435+0000", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8209999999999998e-06 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789170+0000", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037350439999999999 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789333+0000", 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162893 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.789491+0000", 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015807199999999999 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797726+0000", 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082348890000000004 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805320+0000", 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075942850000000001 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.085994+0000", 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28067408599999999 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.086007+0000", 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2286000000000001e-05 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.441 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.2 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.58607065699999, 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30002994399999999, 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.442 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787089+0000", 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.787090+0000", 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.561e-06 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788801+0000", 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001711113 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788814+0000", 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3242e-05 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.788822+0000", 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7109999999999995e-06 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794251+0000", 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054290520000000002 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794407+0000", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015591100000000001 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.794559+0000", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015160600000000001 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801869+0000", 2024-09-13T14:58:13.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007309973 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087105+0000", 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28523667899999999 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087118+0000", 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3096e-05 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.447 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95178 9.2 9:450e6495:::benchmark_data_smithi017_339564_object95177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.45516914699999, 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90794274900000005, 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.448 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95178 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917990+0000", 2024-09-13T14:58:13.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.917993+0000", 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5560000000000002e-06 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918027+0000", 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4338000000000002e-05 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918056+0000", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9411e-05 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918071+0000", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4154999999999999e-05 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087146+0000", 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16907546600000001 2024-09-13T14:58:13.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087156+0000", 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0069e-05 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825063+0000", 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.737906583 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.825933+0000", 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087017099999999999 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95181 9.12 9:4b7fcf1b:::benchmark_data_smithi017_339564_object95180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.455062317, 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92020213900000003, 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95181 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.454 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918097+0000", 2024-09-13T14:58:13.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3300000000000003e-07 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918118+0000", 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0727e-05 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0434999999999999e-05 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918134+0000", 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5589999999999996e-06 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087181+0000", 2024-09-13T14:58:13.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16904727899999999 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087188+0000", 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7519999999999999e-06 2024-09-13T14:58:13.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838221+0000", 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.751033124 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838299+0000", 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7830000000000005e-05 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95184 9.12 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.45501804700001, 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92069096500000003, 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.460 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95184 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918142+0000", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6800000000000001e-07 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918175+0000", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3642000000000001e-05 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918182+0000", 2024-09-13T14:58:13.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2289999999999997e-06 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3920000000000002e-06 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087211+0000", 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16902355999999999 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.087217+0000", 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.835e-06 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838755+0000", 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.75153752500000004 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.838832+0000", 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7414000000000001e-05 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95200 9.18 9:1838bbdd:::benchmark_data_smithi017_339564_object95199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.185063501, 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65754950099999998, 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.465 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95200 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188096+0000", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188098+0000", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4040000000000002e-06 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188129+0000", 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1032999999999999e-05 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188155+0000", 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6322999999999999e-05 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188169+0000", 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3512999999999999e-05 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188244+0000", 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5078000000000002e-05 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.188258+0000", 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.347e-05 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839583+0000", 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.65132497099999997 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839664+0000", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1942999999999998e-05 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.839771+0000", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010627 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842419+0000", 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026485020000000001 2024-09-13T14:58:13.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.842538+0000", 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119218 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845548+0000", 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030097520000000001 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845600+0000", 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2218e-05 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.845645+0000", 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4805999999999998e-05 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123190 10.4 e467/449 10:291845be:::benchmark_data_smithi017_372078_object123189:head v 467'7736, mlcod=467'7734)", 2024-09-13T14:58:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.560899+0000", 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.812260288, 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26569726700000001, 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560899+0000", 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560899+0000", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560900+0000", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000005e-07 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560926+0000", 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6270000000000001e-05 2024-09-13T14:58:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560939+0000", 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3011e-05 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.560947+0000", 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4179999999999997e-06 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563860+0000", 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029129780000000001 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563888+0000", 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7308e-05 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.826498+0000", 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26261044099999997 2024-09-13T14:58:13.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.826596+0000", 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7974999999999996e-05 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123188 10.5 e467/449 10:acb7fc33:::benchmark_data_smithi017_372078_object123187:head v 467'7578, mlcod=467'7576)", 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.563872+0000", 2024-09-13T14:58:13.477 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.809287134, 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25879871700000001, 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563872+0000", 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563872+0000", 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563873+0000", 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5700000000000001e-07 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563908+0000", 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4721999999999997e-05 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563920+0000", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1963999999999999e-05 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.563927+0000", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7910000000000005e-06 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564010+0000", 2024-09-13T14:58:13.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2114999999999996e-05 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564030+0000", 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9996e-05 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568727+0000", 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046975790000000003 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822671+0000", 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25394369300000003 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123193 10.3 e467/449 10:c768659d:::benchmark_data_smithi017_372078_object123192:head v 467'7622, mlcod=467'7619)", 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.564091+0000", 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.809067877, 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25856021800000001, 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.482 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564091+0000", 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564091+0000", 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564092+0000", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8899999999999999e-07 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564199+0000", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010671999999999999 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564209+0000", 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0725e-05 2024-09-13T14:58:13.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564228+0000", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8580000000000002e-05 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564292+0000", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4023999999999996e-05 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.564310+0000", 2024-09-13T14:58:13.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7771e-05 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.571696+0000", 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073863419999999997 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822652+0000", 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25095536699999998 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.486 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123196 10.f e467/449 10:fe61610c:::benchmark_data_smithi017_372078_object123195:head v 467'7807, mlcod=467'7804)", 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.565651+0000", 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.807508422, 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26487944699999999, 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565651+0000", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565651+0000", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565652+0000", 2024-09-13T14:58:13.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7599999999999996e-07 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565695+0000", 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3015000000000001e-05 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565707+0000", 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.208e-05 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.565717+0000", 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0609e-05 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568179+0000", 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024615050000000001 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568201+0000", 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1892e-05 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.830485+0000", 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26228428399999998 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.830530+0000", 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5185999999999997e-05 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.491 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123198 10.f e467/449 10:f78e9f7e:::benchmark_data_smithi017_372078_object123197:head v 467'7808, mlcod=467'7805)", 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.568645+0000", 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.804514118, 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27140502700000002, 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.492 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568645+0000", 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568645+0000", 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568646+0000", 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0800000000000003e-07 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568700+0000", 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3609999999999997e-05 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.568711+0000", 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.149e-05 2024-09-13T14:58:13.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822604+0000", 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25389244300000002 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822759+0000", 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000155458 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822779+0000", 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0412e-05 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.839552+0000", 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016772117999999999 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.840050+0000", 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000498588 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.496 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.37095.0:123201 10.4 e467/449 10:2f495801:::benchmark_data_smithi017_372078_object123200:head v 467'7737, mlcod=467'7734)", 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:56:09.577906+0000", 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 123.795253097, 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25287368599999999, 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.577906+0000", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.577906+0000", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.577908+0000", 2024-09-13T14:58:13.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.404e-06 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.577933+0000", 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5969000000000002e-05 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822691+0000", 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24475724500000001 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822700+0000", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3409999999999992e-06 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822748+0000", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7886999999999997e-05 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.822773+0000", 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4748000000000001e-05 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.830726+0000", 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079529389999999991 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:56:09.830780+0000", 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4153000000000001e-05 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.501 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:58:13.502 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:58:13.502 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:58:13.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:13.536 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:13.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:13.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:13.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:13.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:13.932 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.964 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:13.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:13.974 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:14.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:14.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:14.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:14.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:14.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:14.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:14.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:14.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:14.636 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.673 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.774 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.811 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:14.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:14.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:14.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.049 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:15.077 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:15.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:14 smithi017 ceph-mon[103001]: pgmap v1606: 109 pgs: 109 active+clean; 599 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:58:15.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:14 smithi017 ceph-mon[107556]: pgmap v1606: 109 pgs: 109 active+clean; 599 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:58:15.179 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.216 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:15.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:14 smithi189 ceph-mon[90992]: pgmap v1606: 109 pgs: 109 active+clean; 599 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-13T14:58:15.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.465 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.505 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:15.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:15.607 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:15.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:15.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:15.878 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:15.902 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:58:15.906 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:16.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.142 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.194 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:16.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:16.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:16.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:16.335 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:16.432 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:16.463 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.464 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:16.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:16.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:16.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.751 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:16.853 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.901 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:16.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:17.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.037 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:17.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:17.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:16 smithi017 ceph-mon[103001]: pgmap v1607: 109 pgs: 109 active+clean; 542 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:58:17.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:16 smithi017 ceph-mon[107556]: pgmap v1607: 109 pgs: 109 active+clean; 542 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:58:17.139 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:17.177 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:17.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:16 smithi189 ceph-mon[90992]: pgmap v1607: 109 pgs: 109 active+clean; 542 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:58:17.278 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.308 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:17.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:17.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:17.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:17.654 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:17] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-09-13T14:58:17.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:17.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:17.837 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:17.875 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:17.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.024 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.126 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.168 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.270 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:18.311 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:18.413 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:18.442 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:18.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:18.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:18.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:18.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:249573 10.1 10:847c22fe:::benchmark_data_smithi017_372078_object123269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:58:18.577677+0000", 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.01091395, 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010946539, 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:18.591 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 249573 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577677+0000", 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577677+0000", 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.592 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577676+0000", 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577683+0000", 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1110000000000002e-06 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:18.593 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577695+0000", 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2607e-05 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.577702+0000", 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9040000000000003e-06 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.578028+0000", 2024-09-13T14:58:18.594 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032522399999999999 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.578169+0000", 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000141237 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.578274+0000", 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010470499999999999 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.595 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.581726+0000", 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034523900000000001 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.37095.0:249579 10.d e484/449 10:b8b3a960:::benchmark_data_smithi017_372078_object123275:head v 484'15385, mlcod=484'15383)", 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:58:18.584957+0000", 2024-09-13T14:58:18.596 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0036346249999999998, 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003764285, 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584957+0000", 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584957+0000", 2024-09-13T14:58:18.597 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584956+0000", 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584964+0000", 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4379999999999995e-06 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.598 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584983+0000", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9059000000000002e-05 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.584989+0000", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4609999999999996e-06 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585075+0000", 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5018999999999994e-05 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.599 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585095+0000", 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0109e-05 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.37095.0:249581 10.0 e484/449 10:06199b59:::benchmark_data_smithi017_372078_object123277:head v 484'15578, mlcod=484'15576)", 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:58:18.585002+0000", 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0035886820000000002, 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0037724099999999999, 2024-09-13T14:58:18.600 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585002+0000", 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585002+0000", 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.601 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585002+0000", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585007+0000", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3830000000000002e-06 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585013+0000", 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9830000000000001e-06 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.602 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585016+0000", 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3759999999999999e-06 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585184+0000", 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000167901 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.603 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.585202+0000", 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8386e-05 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.37095.0:249578 10.4 e484/449 10:29f1da4f:::benchmark_data_smithi017_372078_object123274:head v 484'15685, mlcod=484'15682)", 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:58:18.586096+0000", 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0024952759999999998, 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027372920000000001, 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:18.604 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586096+0000", 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586096+0000", 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.605 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586095+0000", 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586105+0000", 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8709999999999992e-06 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.606 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586118+0000", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2554e-05 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586123+0000", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5419999999999996e-06 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586238+0000", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011524 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:18.607 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586256+0000", 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7589999999999999e-05 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_repop(client.37095.0:249575 10.c e484/449 10:34298650:::benchmark_data_smithi017_372078_object123271:head v 484'15760, mlcod=484'15758)", 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:58:18.586146+0000", 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 0.0024453919999999998, 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002740592, 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "started", 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:18.608 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586146+0000", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586146+0000", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586145+0000", 2024-09-13T14:58:18.609 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586150+0000", 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2529999999999998e-06 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586156+0000", 2024-09-13T14:58:18.610 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4699999999999999e-06 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.586160+0000", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4989999999999999e-06 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.587543+0000", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00138349 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:58:18.587561+0000", 2024-09-13T14:58:18.611 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7561e-05 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: ], 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 5 2024-09-13T14:58:18.612 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:18.683 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:18.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:18.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.845 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:18.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.881 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:58:18.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:18.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:18.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:19.097 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:18 smithi017 ceph-mon[103001]: pgmap v1608: 109 pgs: 109 active+clean; 523 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:18 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:18 smithi017 ceph-mon[107556]: pgmap v1608: 109 pgs: 109 active+clean; 523 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:18 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:19.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:18 smithi189 ceph-mon[90992]: pgmap v1608: 109 pgs: 109 active+clean; 523 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:19.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:18 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:19.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:19.352 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T14:58:19.357 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T14:58:19.380 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:58:19.411 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:19.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:19.648 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.711 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:19.836 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:19.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:19.949 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.992 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:20.094 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.133 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.162 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:19 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:20.163 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:19 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:20.163 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:19 smithi189 ceph-mon[90992]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:58:20.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:20.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:20.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[103001]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:58:20.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:20.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1328094458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:20.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:19 smithi017 ceph-mon[107556]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:58:20.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:20.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:20.374 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:20.409 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:20.459 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:20.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:20.537 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:20.559 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1609,"stamp":"2024-09-13T14:58:18.877751+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":240588773,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":5969,"num_read_kb":12982,"num_write":249912,"num_write_kb":8095888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":103,"num_bytes_recovered":8721938,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120003,"ondisk_log_size":120003,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8176432,"kb_used_data":860080,"kb_used_omap":44,"kb_used_meta":7316051,"kb_avail":741784784,"statfs":{"total":767960285184,"available":759587618816,"internally_reserved":0,"allocated":880721920,"data_stored":860477487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7491636471},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,4,3,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-909246464,"num_objects":-13874,"num_object_clones":0,"num_object_copies":-41622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-13874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13874,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.061806"},"pg_stats":[{"pgid":"1.11","version":"82'100","reported_seq":303,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821272+0000","last_change":"2024-09-13T14:56:00.831620+0000","last_active":"2024-09-13T14:58:07.821272+0000","last_peered":"2024-09-13T14:58:07.821272+0000","last_clean":"2024-09-13T14:58:07.821272+0000","last_became_active":"2024-09-13T14:56:00.676063+0000","last_became_peered":"2024-09-13T14:56:00.676063+0000","last_unstale":"2024-09-13T14:58:07.821272+0000","last_undegraded":"2024-09-13T14:58:07.821272+0000","last_fullsized":"2024-09-13T14:58:07.821272+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.831575+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.831575+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:29:03.735233+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":857,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.824820+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T14:58:07.824820+0000","last_peered":"2024-09-13T14:58:07.824820+0000","last_clean":"2024-09-13T14:58:07.824820+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T14:58:07.824820+0000","last_undegraded":"2024-09-13T14:58:07.824820+0000","last_fullsized":"2024-09-13T14:58:07.824820+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":482,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821742+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T14:58:06.821742+0000","last_peered":"2024-09-13T14:58:06.821742+0000","last_clean":"2024-09-13T14:58:06.821742+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T14:58:06.821742+0000","last_undegraded":"2024-09-13T14:58:06.821742+0000","last_fullsized":"2024-09-13T14:58:06.821742+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":37,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818234+0000","last_change":"2024-09-13T14:56:04.574886+0000","last_active":"2024-09-13T14:58:06.818234+0000","last_peered":"2024-09-13T14:58:06.818234+0000","last_clean":"2024-09-13T14:58:06.818234+0000","last_became_active":"2024-09-13T14:56:00.674692+0000","last_became_peered":"2024-09-13T14:56:00.674692+0000","last_unstale":"2024-09-13T14:58:06.818234+0000","last_undegraded":"2024-09-13T14:58:06.818234+0000","last_fullsized":"2024-09-13T14:58:06.818234+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.574787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.574787+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:30.673194+0000","scrub_duration":0.0020770139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818943+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T14:58:06.818943+0000","last_peered":"2024-09-13T14:58:06.818943+0000","last_clean":"2024-09-13T14:58:06.818943+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T14:58:06.818943+0000","last_undegraded":"2024-09-13T14:58:06.818943+0000","last_fullsized":"2024-09-13T14:58:06.818943+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821518+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T14:58:07.821518+0000","last_peered":"2024-09-13T14:58:07.821518+0000","last_clean":"2024-09-13T14:58:07.821518+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T14:58:07.821518+0000","last_undegraded":"2024-09-13T14:58:07.821518+0000","last_fullsized":"2024-09-13T14:58:07.821518+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":694,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.829365+0000","last_change":"2024-09-13T14:56:01.351509+0000","last_active":"2024-09-13T14:58:07.829365+0000","last_peered":"2024-09-13T14:58:07.829365+0000","last_clean":"2024-09-13T14:58:07.829365+0000","last_became_active":"2024-09-13T14:56:00.675249+0000","last_became_peered":"2024-09-13T14:56:00.675249+0000","last_unstale":"2024-09-13T14:58:07.829365+0000","last_undegraded":"2024-09-13T14:58:07.829365+0000","last_fullsized":"2024-09-13T14:58:07.829365+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.351298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.351298+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:23.442259+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.830989+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T14:58:07.830989+0000","last_peered":"2024-09-13T14:58:07.830989+0000","last_clean":"2024-09-13T14:58:07.830989+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T14:58:07.830989+0000","last_undegraded":"2024-09-13T14:58:07.830989+0000","last_fullsized":"2024-09-13T14:58:07.830989+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":451,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822004+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T14:58:06.822004+0000","last_peered":"2024-09-13T14:58:06.822004+0000","last_clean":"2024-09-13T14:58:06.822004+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T14:58:06.822004+0000","last_undegraded":"2024-09-13T14:58:06.822004+0000","last_fullsized":"2024-09-13T14:58:06.822004+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":37,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818029+0000","last_change":"2024-09-13T14:56:01.635272+0000","last_active":"2024-09-13T14:58:06.818029+0000","last_peered":"2024-09-13T14:58:06.818029+0000","last_clean":"2024-09-13T14:58:06.818029+0000","last_became_active":"2024-09-13T14:56:00.667926+0000","last_became_peered":"2024-09-13T14:56:00.667926+0000","last_unstale":"2024-09-13T14:58:06.818029+0000","last_undegraded":"2024-09-13T14:58:06.818029+0000","last_fullsized":"2024-09-13T14:58:06.818029+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.635189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.635189+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:28.334884+0000","scrub_duration":0.010355512000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":857,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826089+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T14:58:07.826089+0000","last_peered":"2024-09-13T14:58:07.826089+0000","last_clean":"2024-09-13T14:58:07.826089+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T14:58:07.826089+0000","last_undegraded":"2024-09-13T14:58:07.826089+0000","last_fullsized":"2024-09-13T14:58:07.826089+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":513,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.827586+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T14:58:06.827586+0000","last_peered":"2024-09-13T14:58:06.827586+0000","last_clean":"2024-09-13T14:58:06.827586+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T14:58:06.827586+0000","last_undegraded":"2024-09-13T14:58:06.827586+0000","last_fullsized":"2024-09-13T14:58:06.827586+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":37,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.820725+0000","last_change":"2024-09-13T14:56:00.915923+0000","last_active":"2024-09-13T14:58:06.820725+0000","last_peered":"2024-09-13T14:58:06.820725+0000","last_clean":"2024-09-13T14:58:06.820725+0000","last_became_active":"2024-09-13T14:56:00.671638+0000","last_became_peered":"2024-09-13T14:56:00.671638+0000","last_unstale":"2024-09-13T14:58:06.820725+0000","last_undegraded":"2024-09-13T14:58:06.820725+0000","last_fullsized":"2024-09-13T14:58:06.820725+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.915835+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.915835+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:52.043909+0000","scrub_duration":0.010872074000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"484'15451","reported_seq":15492,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.533684+0000","last_change":"2024-09-13T14:54:28.814385+0000","last_active":"2024-09-13T14:58:18.533684+0000","last_peered":"2024-09-13T14:58:18.533684+0000","last_clean":"2024-09-13T14:58:18.533684+0000","last_became_active":"2024-09-13T14:54:28.814061+0000","last_became_peered":"2024-09-13T14:54:28.814061+0000","last_unstale":"2024-09-13T14:58:18.533684+0000","last_undegraded":"2024-09-13T14:58:18.533684+0000","last_fullsized":"2024-09-13T14:58:18.533684+0000","mapping_epoch":449,"log_start":"464'6800","ondisk_log_start":"464'6800","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:10.454908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15451,"num_write_kb":500736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":505,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821459+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T14:58:06.821459+0000","last_peered":"2024-09-13T14:58:06.821459+0000","last_clean":"2024-09-13T14:58:06.821459+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T14:58:06.821459+0000","last_undegraded":"2024-09-13T14:58:06.821459+0000","last_fullsized":"2024-09-13T14:58:06.821459+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":396,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818066+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T14:58:06.818066+0000","last_peered":"2024-09-13T14:58:06.818066+0000","last_clean":"2024-09-13T14:58:06.818066+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T14:58:06.818066+0000","last_undegraded":"2024-09-13T14:58:06.818066+0000","last_fullsized":"2024-09-13T14:58:06.818066+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":727,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.824716+0000","last_change":"2024-09-13T14:56:00.697476+0000","last_active":"2024-09-13T14:58:07.824716+0000","last_peered":"2024-09-13T14:58:07.824716+0000","last_clean":"2024-09-13T14:58:07.824716+0000","last_became_active":"2024-09-13T14:56:00.680491+0000","last_became_peered":"2024-09-13T14:56:00.680491+0000","last_unstale":"2024-09-13T14:58:07.824716+0000","last_undegraded":"2024-09-13T14:58:07.824716+0000","last_fullsized":"2024-09-13T14:58:07.824716+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:56.869434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:56.869434+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:32.392000+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.7","version":"484'15691","reported_seq":15734,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.985012+0000","last_change":"2024-09-13T14:54:28.839869+0000","last_active":"2024-09-13T14:58:17.985012+0000","last_peered":"2024-09-13T14:58:17.985012+0000","last_clean":"2024-09-13T14:58:17.985012+0000","last_became_active":"2024-09-13T14:54:28.839616+0000","last_became_peered":"2024-09-13T14:54:28.839616+0000","last_unstale":"2024-09-13T14:58:17.985012+0000","last_undegraded":"2024-09-13T14:58:17.985012+0000","last_fullsized":"2024-09-13T14:58:17.985012+0000","mapping_epoch":449,"log_start":"469'8676","ondisk_log_start":"469'8676","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7015,"log_dups_size":0,"ondisk_log_size":7015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:30:06.859854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15691,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":495,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821176+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T14:58:06.821176+0000","last_peered":"2024-09-13T14:58:06.821176+0000","last_clean":"2024-09-13T14:58:06.821176+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T14:58:06.821176+0000","last_undegraded":"2024-09-13T14:58:06.821176+0000","last_fullsized":"2024-09-13T14:58:06.821176+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821809+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T14:58:06.821809+0000","last_peered":"2024-09-13T14:58:06.821809+0000","last_clean":"2024-09-13T14:58:06.821809+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T14:58:06.821809+0000","last_undegraded":"2024-09-13T14:58:06.821809+0000","last_fullsized":"2024-09-13T14:58:06.821809+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":512,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.831144+0000","last_change":"2024-09-13T14:56:02.698903+0000","last_active":"2024-09-13T14:58:06.831144+0000","last_peered":"2024-09-13T14:58:06.831144+0000","last_clean":"2024-09-13T14:58:06.831144+0000","last_became_active":"2024-09-13T14:56:00.691242+0000","last_became_peered":"2024-09-13T14:56:00.691242+0000","last_unstale":"2024-09-13T14:58:06.831144+0000","last_undegraded":"2024-09-13T14:58:06.831144+0000","last_fullsized":"2024-09-13T14:58:06.831144+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.698817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.698817+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:11.424244+0000","scrub_duration":0.0077410969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"484'15637","reported_seq":15718,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.694340+0000","last_change":"2024-09-13T14:54:28.841842+0000","last_active":"2024-09-13T14:58:17.694340+0000","last_peered":"2024-09-13T14:58:17.694340+0000","last_clean":"2024-09-13T14:58:17.694340+0000","last_became_active":"2024-09-13T14:54:28.841527+0000","last_became_peered":"2024-09-13T14:54:28.841527+0000","last_unstale":"2024-09-13T14:58:17.694340+0000","last_undegraded":"2024-09-13T14:58:17.694340+0000","last_fullsized":"2024-09-13T14:58:17.694340+0000","mapping_epoch":449,"log_start":"468'8456","ondisk_log_start":"468'8456","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7181,"log_dups_size":0,"ondisk_log_size":7181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:55:39.914494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15637,"num_write_kb":508480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":449,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821501+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T14:58:06.821501+0000","last_peered":"2024-09-13T14:58:06.821501+0000","last_clean":"2024-09-13T14:58:06.821501+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T14:58:06.821501+0000","last_undegraded":"2024-09-13T14:58:06.821501+0000","last_fullsized":"2024-09-13T14:58:06.821501+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":464,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.827127+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T14:58:06.827127+0000","last_peered":"2024-09-13T14:58:06.827127+0000","last_clean":"2024-09-13T14:58:06.827127+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T14:58:06.827127+0000","last_undegraded":"2024-09-13T14:58:06.827127+0000","last_fullsized":"2024-09-13T14:58:06.827127+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":37,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822682+0000","last_change":"2024-09-13T14:56:05.805624+0000","last_active":"2024-09-13T14:58:06.822682+0000","last_peered":"2024-09-13T14:58:06.822682+0000","last_clean":"2024-09-13T14:58:06.822682+0000","last_became_active":"2024-09-13T14:56:00.676524+0000","last_became_peered":"2024-09-13T14:56:00.676524+0000","last_unstale":"2024-09-13T14:58:06.822682+0000","last_undegraded":"2024-09-13T14:58:06.822682+0000","last_fullsized":"2024-09-13T14:58:06.822682+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:05.805551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:05.805551+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:13.686450+0000","scrub_duration":0.028580083999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.5","version":"484'15294","reported_seq":15375,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.726521+0000","last_change":"2024-09-13T14:54:28.842095+0000","last_active":"2024-09-13T14:58:17.726521+0000","last_peered":"2024-09-13T14:58:17.726521+0000","last_clean":"2024-09-13T14:58:17.726521+0000","last_became_active":"2024-09-13T14:54:28.841519+0000","last_became_peered":"2024-09-13T14:54:28.841519+0000","last_unstale":"2024-09-13T14:58:17.726521+0000","last_undegraded":"2024-09-13T14:58:17.726521+0000","last_fullsized":"2024-09-13T14:58:17.726521+0000","mapping_epoch":449,"log_start":"468'8133","ondisk_log_start":"468'8133","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7161,"log_dups_size":0,"ondisk_log_size":7161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:37.400416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15294,"num_write_kb":497536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":412,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818545+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T14:58:06.818545+0000","last_peered":"2024-09-13T14:58:06.818545+0000","last_clean":"2024-09-13T14:58:06.818545+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T14:58:06.818545+0000","last_undegraded":"2024-09-13T14:58:06.818545+0000","last_fullsized":"2024-09-13T14:58:06.818545+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821931+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T14:58:07.821931+0000","last_peered":"2024-09-13T14:58:07.821931+0000","last_clean":"2024-09-13T14:58:07.821931+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T14:58:07.821931+0000","last_undegraded":"2024-09-13T14:58:07.821931+0000","last_fullsized":"2024-09-13T14:58:07.821931+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"484'15678","reported_seq":15760,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:15.980735+0000","last_change":"2024-09-13T14:54:28.810318+0000","last_active":"2024-09-13T14:58:15.980735+0000","last_peered":"2024-09-13T14:58:15.980735+0000","last_clean":"2024-09-13T14:58:15.980735+0000","last_became_active":"2024-09-13T14:54:28.809747+0000","last_became_peered":"2024-09-13T14:54:28.809747+0000","last_unstale":"2024-09-13T14:58:15.980735+0000","last_undegraded":"2024-09-13T14:58:15.980735+0000","last_fullsized":"2024-09-13T14:58:15.980735+0000","mapping_epoch":449,"log_start":"468'8100","ondisk_log_start":"468'8100","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:20.591218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15678,"num_write_kb":513024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":595,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821835+0000","last_change":"2024-09-13T14:30:41.431794+0000","last_active":"2024-09-13T14:58:06.821835+0000","last_peered":"2024-09-13T14:58:06.821835+0000","last_clean":"2024-09-13T14:58:06.821835+0000","last_became_active":"2024-09-13T14:30:11.460020+0000","last_became_peered":"2024-09-13T14:30:11.460020+0000","last_unstale":"2024-09-13T14:58:06.821835+0000","last_undegraded":"2024-09-13T14:58:06.821835+0000","last_fullsized":"2024-09-13T14:58:06.821835+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:30:41.431708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:30:41.431708+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:16:30.072719+0000","scrub_duration":0.0056898210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":889,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.841473+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T14:58:07.841473+0000","last_peered":"2024-09-13T14:58:07.841473+0000","last_clean":"2024-09-13T14:58:07.841473+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T14:58:07.841473+0000","last_undegraded":"2024-09-13T14:58:07.841473+0000","last_fullsized":"2024-09-13T14:58:07.841473+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822586+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T14:58:06.822586+0000","last_peered":"2024-09-13T14:58:06.822586+0000","last_clean":"2024-09-13T14:58:06.822586+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T14:58:06.822586+0000","last_undegraded":"2024-09-13T14:58:06.822586+0000","last_fullsized":"2024-09-13T14:58:06.822586+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"484'15442","reported_seq":15485,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.504400+0000","last_change":"2024-09-13T14:54:28.840698+0000","last_active":"2024-09-13T14:58:18.504400+0000","last_peered":"2024-09-13T14:58:18.504400+0000","last_clean":"2024-09-13T14:58:18.504400+0000","last_became_active":"2024-09-13T14:54:28.840455+0000","last_became_peered":"2024-09-13T14:54:28.840455+0000","last_unstale":"2024-09-13T14:58:18.504400+0000","last_undegraded":"2024-09-13T14:58:18.504400+0000","last_fullsized":"2024-09-13T14:58:18.504400+0000","mapping_epoch":449,"log_start":"464'6800","ondisk_log_start":"464'6800","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8642,"log_dups_size":0,"ondisk_log_size":8642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:50.324481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12845056,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15442,"num_write_kb":500416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":309,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818135+0000","last_change":"2024-09-13T14:49:50.703153+0000","last_active":"2024-09-13T14:58:06.818135+0000","last_peered":"2024-09-13T14:58:06.818135+0000","last_clean":"2024-09-13T14:58:06.818135+0000","last_became_active":"2024-09-13T14:40:44.705901+0000","last_became_peered":"2024-09-13T14:40:44.705901+0000","last_unstale":"2024-09-13T14:58:06.818135+0000","last_undegraded":"2024-09-13T14:58:06.818135+0000","last_fullsized":"2024-09-13T14:58:06.818135+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:50.703079+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:50.703079+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:57.719511+0000","scrub_duration":0.001916013,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":485,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.827005+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T14:58:06.827005+0000","last_peered":"2024-09-13T14:58:06.827005+0000","last_clean":"2024-09-13T14:58:06.827005+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T14:58:06.827005+0000","last_undegraded":"2024-09-13T14:58:06.827005+0000","last_fullsized":"2024-09-13T14:58:06.827005+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":196,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822949+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T14:58:06.822949+0000","last_peered":"2024-09-13T14:58:06.822949+0000","last_clean":"2024-09-13T14:58:06.822949+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T14:58:06.822949+0000","last_undegraded":"2024-09-13T14:58:06.822949+0000","last_fullsized":"2024-09-13T14:58:06.822949+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":691,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.829506+0000","last_change":"2024-09-13T14:56:00.671941+0000","last_active":"2024-09-13T14:58:07.829506+0000","last_peered":"2024-09-13T14:58:07.829506+0000","last_clean":"2024-09-13T14:58:07.829506+0000","last_became_active":"2024-09-13T14:56:00.668042+0000","last_became_peered":"2024-09-13T14:56:00.668042+0000","last_unstale":"2024-09-13T14:58:07.829506+0000","last_undegraded":"2024-09-13T14:58:07.829506+0000","last_fullsized":"2024-09-13T14:58:07.829506+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.375774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.375774+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:18.945468+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"484'15540","reported_seq":15583,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.989547+0000","last_change":"2024-09-13T14:54:28.816802+0000","last_active":"2024-09-13T14:58:17.989547+0000","last_peered":"2024-09-13T14:58:17.989547+0000","last_clean":"2024-09-13T14:58:17.989547+0000","last_became_active":"2024-09-13T14:54:28.816143+0000","last_became_peered":"2024-09-13T14:54:28.816143+0000","last_unstale":"2024-09-13T14:58:17.989547+0000","last_undegraded":"2024-09-13T14:58:17.989547+0000","last_fullsized":"2024-09-13T14:58:17.989547+0000","mapping_epoch":449,"log_start":"469'8467","ondisk_log_start":"469'8467","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:25:19.767169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14417920,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15540,"num_write_kb":504320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821561+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T14:58:06.821561+0000","last_peered":"2024-09-13T14:58:06.821561+0000","last_clean":"2024-09-13T14:58:06.821561+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T14:58:06.821561+0000","last_undegraded":"2024-09-13T14:58:06.821561+0000","last_fullsized":"2024-09-13T14:58:06.821561+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.823922+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T14:58:06.823922+0000","last_peered":"2024-09-13T14:58:06.823922+0000","last_clean":"2024-09-13T14:58:06.823922+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T14:58:06.823922+0000","last_undegraded":"2024-09-13T14:58:06.823922+0000","last_fullsized":"2024-09-13T14:58:06.823922+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"484'15357","reported_seq":15400,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.168229+0000","last_change":"2024-09-13T14:54:28.815313+0000","last_active":"2024-09-13T14:58:18.168229+0000","last_peered":"2024-09-13T14:58:18.168229+0000","last_clean":"2024-09-13T14:58:18.168229+0000","last_became_active":"2024-09-13T14:54:28.815029+0000","last_became_peered":"2024-09-13T14:54:28.815029+0000","last_unstale":"2024-09-13T14:58:18.168229+0000","last_undegraded":"2024-09-13T14:58:18.168229+0000","last_fullsized":"2024-09-13T14:58:18.168229+0000","mapping_epoch":449,"log_start":"471'9135","ondisk_log_start":"471'9135","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:29.140771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14745600,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15357,"num_write_kb":498624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":302,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818917+0000","last_change":"2024-09-13T14:49:54.735073+0000","last_active":"2024-09-13T14:58:06.818917+0000","last_peered":"2024-09-13T14:58:06.818917+0000","last_clean":"2024-09-13T14:58:06.818917+0000","last_became_active":"2024-09-13T14:40:44.706687+0000","last_became_peered":"2024-09-13T14:40:44.706687+0000","last_unstale":"2024-09-13T14:58:06.818917+0000","last_undegraded":"2024-09-13T14:58:06.818917+0000","last_fullsized":"2024-09-13T14:58:06.818917+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:49:54.735012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:49:54.735012+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:08.019427+0000","scrub_duration":0.001458106,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":859,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.822823+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T14:58:07.822823+0000","last_peered":"2024-09-13T14:58:07.822823+0000","last_clean":"2024-09-13T14:58:07.822823+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T14:58:07.822823+0000","last_undegraded":"2024-09-13T14:58:07.822823+0000","last_fullsized":"2024-09-13T14:58:07.822823+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":821,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.827084+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T14:58:07.827084+0000","last_peered":"2024-09-13T14:58:07.827084+0000","last_clean":"2024-09-13T14:58:07.827084+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T14:58:07.827084+0000","last_undegraded":"2024-09-13T14:58:07.827084+0000","last_fullsized":"2024-09-13T14:58:07.827084+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"484'15568","reported_seq":15611,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.977967+0000","last_change":"2024-09-13T14:54:28.817891+0000","last_active":"2024-09-13T14:58:17.977967+0000","last_peered":"2024-09-13T14:58:17.977967+0000","last_clean":"2024-09-13T14:58:17.977967+0000","last_became_active":"2024-09-13T14:54:28.817390+0000","last_became_peered":"2024-09-13T14:54:28.817390+0000","last_unstale":"2024-09-13T14:58:17.977967+0000","last_undegraded":"2024-09-13T14:58:17.977967+0000","last_fullsized":"2024-09-13T14:58:17.977967+0000","mapping_epoch":449,"log_start":"469'8500","ondisk_log_start":"469'8500","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:33:30.308516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13631488,"num_objects":208,"num_object_clones":0,"num_object_copies":624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15568,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":626,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821777+0000","last_change":"2024-09-13T14:55:57.906683+0000","last_active":"2024-09-13T14:58:06.821777+0000","last_peered":"2024-09-13T14:58:06.821777+0000","last_clean":"2024-09-13T14:58:06.821777+0000","last_became_active":"2024-09-13T14:55:54.613915+0000","last_became_peered":"2024-09-13T14:55:54.613915+0000","last_unstale":"2024-09-13T14:58:06.821777+0000","last_undegraded":"2024-09-13T14:58:06.821777+0000","last_fullsized":"2024-09-13T14:58:06.821777+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.906630+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.906630+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:03.677753+0000","scrub_duration":0.001255616,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":1008,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818535+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T14:58:06.818535+0000","last_peered":"2024-09-13T14:58:06.818535+0000","last_clean":"2024-09-13T14:58:06.818535+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T14:58:06.818535+0000","last_undegraded":"2024-09-13T14:58:06.818535+0000","last_fullsized":"2024-09-13T14:58:06.818535+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":813,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.823461+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T14:58:07.823461+0000","last_peered":"2024-09-13T14:58:07.823461+0000","last_clean":"2024-09-13T14:58:07.823461+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T14:58:07.823461+0000","last_undegraded":"2024-09-13T14:58:07.823461+0000","last_fullsized":"2024-09-13T14:58:07.823461+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":450,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.827448+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T14:58:06.827448+0000","last_peered":"2024-09-13T14:58:06.827448+0000","last_clean":"2024-09-13T14:58:06.827448+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T14:58:06.827448+0000","last_undegraded":"2024-09-13T14:58:06.827448+0000","last_fullsized":"2024-09-13T14:58:06.827448+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"484'15494","reported_seq":15576,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.693793+0000","last_change":"2024-09-13T14:54:28.810322+0000","last_active":"2024-09-13T14:58:17.693793+0000","last_peered":"2024-09-13T14:58:17.693793+0000","last_clean":"2024-09-13T14:58:17.693793+0000","last_became_active":"2024-09-13T14:54:28.809987+0000","last_became_peered":"2024-09-13T14:54:28.809987+0000","last_unstale":"2024-09-13T14:58:17.693793+0000","last_undegraded":"2024-09-13T14:58:17.693793+0000","last_fullsized":"2024-09-13T14:58:17.693793+0000","mapping_epoch":449,"log_start":"468'8302","ondisk_log_start":"468'8302","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7192,"log_dups_size":0,"ondisk_log_size":7192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:16.785375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13893632,"num_objects":212,"num_object_clones":0,"num_object_copies":636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15494,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":688,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.829596+0000","last_change":"2024-09-13T14:56:00.322316+0000","last_active":"2024-09-13T14:58:07.829596+0000","last_peered":"2024-09-13T14:58:07.829596+0000","last_clean":"2024-09-13T14:58:07.829596+0000","last_became_active":"2024-09-13T14:55:54.607589+0000","last_became_peered":"2024-09-13T14:55:54.607589+0000","last_unstale":"2024-09-13T14:58:07.829596+0000","last_undegraded":"2024-09-13T14:58:07.829596+0000","last_fullsized":"2024-09-13T14:58:07.829596+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.322230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.322230+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:29:10.503431+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":896,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821435+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T14:58:06.821435+0000","last_peered":"2024-09-13T14:58:06.821435+0000","last_clean":"2024-09-13T14:58:06.821435+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T14:58:06.821435+0000","last_undegraded":"2024-09-13T14:58:06.821435+0000","last_fullsized":"2024-09-13T14:58:06.821435+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821112+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T14:58:07.821112+0000","last_peered":"2024-09-13T14:58:07.821112+0000","last_clean":"2024-09-13T14:58:07.821112+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T14:58:07.821112+0000","last_undegraded":"2024-09-13T14:58:07.821112+0000","last_fullsized":"2024-09-13T14:58:07.821112+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.830796+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T14:58:07.830796+0000","last_peered":"2024-09-13T14:58:07.830796+0000","last_clean":"2024-09-13T14:58:07.830796+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T14:58:07.830796+0000","last_undegraded":"2024-09-13T14:58:07.830796+0000","last_fullsized":"2024-09-13T14:58:07.830796+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"484'15774","reported_seq":15817,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.531064+0000","last_change":"2024-09-13T14:54:28.810442+0000","last_active":"2024-09-13T14:58:18.531064+0000","last_peered":"2024-09-13T14:58:18.531064+0000","last_clean":"2024-09-13T14:58:18.531064+0000","last_became_active":"2024-09-13T14:54:28.810148+0000","last_became_peered":"2024-09-13T14:54:28.810148+0000","last_unstale":"2024-09-13T14:58:18.531064+0000","last_undegraded":"2024-09-13T14:58:18.531064+0000","last_fullsized":"2024-09-13T14:58:18.531064+0000","mapping_epoch":449,"log_start":"465'7200","ondisk_log_start":"465'7200","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":8574,"log_dups_size":0,"ondisk_log_size":8574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:15:56.579491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13369344,"num_objects":204,"num_object_clones":0,"num_object_copies":612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15774,"num_write_kb":511296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":655,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.820723+0000","last_change":"2024-09-13T14:55:58.908251+0000","last_active":"2024-09-13T14:58:06.820723+0000","last_peered":"2024-09-13T14:58:06.820723+0000","last_clean":"2024-09-13T14:58:06.820723+0000","last_became_active":"2024-09-13T14:55:54.619910+0000","last_became_peered":"2024-09-13T14:55:54.619910+0000","last_unstale":"2024-09-13T14:58:06.820723+0000","last_undegraded":"2024-09-13T14:58:06.820723+0000","last_fullsized":"2024-09-13T14:58:06.820723+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.908152+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.908152+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:01.217177+0000","scrub_duration":0.0048598549999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":840,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.823321+0000","last_change":"2024-09-13T14:53:09.769147+0000","last_active":"2024-09-13T14:58:07.823321+0000","last_peered":"2024-09-13T14:58:07.823321+0000","last_clean":"2024-09-13T14:58:07.823321+0000","last_became_active":"2024-09-13T14:53:09.768166+0000","last_became_peered":"2024-09-13T14:53:09.768166+0000","last_unstale":"2024-09-13T14:58:07.823321+0000","last_undegraded":"2024-09-13T14:58:07.823321+0000","last_fullsized":"2024-09-13T14:58:07.823321+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:25.349412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":799,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.829007+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T14:58:07.829007+0000","last_peered":"2024-09-13T14:58:07.829007+0000","last_clean":"2024-09-13T14:58:07.829007+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T14:58:07.829007+0000","last_undegraded":"2024-09-13T14:58:07.829007+0000","last_fullsized":"2024-09-13T14:58:07.829007+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":485,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.833666+0000","last_change":"2024-09-13T14:58:07.832480+0000","last_active":"2024-09-13T14:58:07.833666+0000","last_peered":"2024-09-13T14:58:07.833666+0000","last_clean":"2024-09-13T14:58:07.833666+0000","last_became_active":"2024-09-13T14:58:07.831527+0000","last_became_peered":"2024-09-13T14:58:07.831527+0000","last_unstale":"2024-09-13T14:58:07.833666+0000","last_undegraded":"2024-09-13T14:58:07.833666+0000","last_fullsized":"2024-09-13T14:58:07.833666+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:25.525827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"484'15465","reported_seq":15508,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.930721+0000","last_change":"2024-09-13T14:54:28.816690+0000","last_active":"2024-09-13T14:58:17.930721+0000","last_peered":"2024-09-13T14:58:17.930721+0000","last_clean":"2024-09-13T14:58:17.930721+0000","last_became_active":"2024-09-13T14:54:28.816334+0000","last_became_peered":"2024-09-13T14:54:28.816334+0000","last_unstale":"2024-09-13T14:58:17.930721+0000","last_undegraded":"2024-09-13T14:58:17.930721+0000","last_fullsized":"2024-09-13T14:58:17.930721+0000","mapping_epoch":449,"log_start":"469'8400","ondisk_log_start":"469'8400","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:45.707837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15007744,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15465,"num_write_kb":502208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":342,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818911+0000","last_change":"2024-09-13T14:56:00.644560+0000","last_active":"2024-09-13T14:58:06.818911+0000","last_peered":"2024-09-13T14:58:06.818911+0000","last_clean":"2024-09-13T14:58:06.818911+0000","last_became_active":"2024-09-13T14:55:54.621234+0000","last_became_peered":"2024-09-13T14:55:54.621234+0000","last_unstale":"2024-09-13T14:58:06.818911+0000","last_undegraded":"2024-09-13T14:58:06.818911+0000","last_fullsized":"2024-09-13T14:58:06.818911+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.644466+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.644466+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.375763+0000","scrub_duration":0.0093267220000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1099,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.819048+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T14:58:06.819048+0000","last_peered":"2024-09-13T14:58:06.819048+0000","last_clean":"2024-09-13T14:58:06.819048+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T14:58:06.819048+0000","last_undegraded":"2024-09-13T14:58:06.819048+0000","last_fullsized":"2024-09-13T14:58:06.819048+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":202,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822372+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T14:58:06.822372+0000","last_peered":"2024-09-13T14:58:06.822372+0000","last_clean":"2024-09-13T14:58:06.822372+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T14:58:06.822372+0000","last_undegraded":"2024-09-13T14:58:06.822372+0000","last_fullsized":"2024-09-13T14:58:06.822372+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":820,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826076+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T14:58:07.826076+0000","last_peered":"2024-09-13T14:58:07.826076+0000","last_clean":"2024-09-13T14:58:07.826076+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T14:58:07.826076+0000","last_undegraded":"2024-09-13T14:58:07.826076+0000","last_fullsized":"2024-09-13T14:58:07.826076+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"484'15751","reported_seq":15794,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.317320+0000","last_change":"2024-09-13T14:54:28.814568+0000","last_active":"2024-09-13T14:58:18.317320+0000","last_peered":"2024-09-13T14:58:18.317320+0000","last_clean":"2024-09-13T14:58:18.317320+0000","last_became_active":"2024-09-13T14:54:28.814244+0000","last_became_peered":"2024-09-13T14:54:28.814244+0000","last_unstale":"2024-09-13T14:58:18.317320+0000","last_undegraded":"2024-09-13T14:58:18.317320+0000","last_fullsized":"2024-09-13T14:58:18.317320+0000","mapping_epoch":449,"log_start":"469'8542","ondisk_log_start":"469'8542","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:12.358506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13697024,"num_objects":209,"num_object_clones":0,"num_object_copies":627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15751,"num_write_kb":510720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":503,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.831068+0000","last_change":"2024-09-13T14:55:59.724453+0000","last_active":"2024-09-13T14:58:06.831068+0000","last_peered":"2024-09-13T14:58:06.831068+0000","last_clean":"2024-09-13T14:58:06.831068+0000","last_became_active":"2024-09-13T14:55:54.613965+0000","last_became_peered":"2024-09-13T14:55:54.613965+0000","last_unstale":"2024-09-13T14:58:06.831068+0000","last_undegraded":"2024-09-13T14:58:06.831068+0000","last_fullsized":"2024-09-13T14:58:06.831068+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.723985+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.723985+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:40:11.061534+0000","scrub_duration":0.0085871660000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":429,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818828+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T14:58:06.818828+0000","last_peered":"2024-09-13T14:58:06.818828+0000","last_clean":"2024-09-13T14:58:06.818828+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T14:58:06.818828+0000","last_undegraded":"2024-09-13T14:58:06.818828+0000","last_fullsized":"2024-09-13T14:58:06.818828+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1483,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826267+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T14:58:07.826267+0000","last_peered":"2024-09-13T14:58:07.826267+0000","last_clean":"2024-09-13T14:58:07.826267+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T14:58:07.826267+0000","last_undegraded":"2024-09-13T14:58:07.826267+0000","last_fullsized":"2024-09-13T14:58:07.826267+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":801,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.828162+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T14:58:07.828162+0000","last_peered":"2024-09-13T14:58:07.828162+0000","last_clean":"2024-09-13T14:58:07.828162+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T14:58:07.828162+0000","last_undegraded":"2024-09-13T14:58:07.828162+0000","last_fullsized":"2024-09-13T14:58:07.828162+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"484'15359","reported_seq":15403,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.992377+0000","last_change":"2024-09-13T14:54:28.816546+0000","last_active":"2024-09-13T14:58:17.992377+0000","last_peered":"2024-09-13T14:58:17.992377+0000","last_clean":"2024-09-13T14:58:17.992377+0000","last_became_active":"2024-09-13T14:54:28.816165+0000","last_became_peered":"2024-09-13T14:54:28.816165+0000","last_unstale":"2024-09-13T14:58:17.992377+0000","last_undegraded":"2024-09-13T14:58:17.992377+0000","last_fullsized":"2024-09-13T14:58:17.992377+0000","mapping_epoch":449,"log_start":"468'8283","ondisk_log_start":"468'8283","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:43.867728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12320792,"num_objects":189,"num_object_clones":0,"num_object_copies":567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":189,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15359,"num_write_kb":497473,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":523,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.827426+0000","last_change":"2024-09-13T14:56:00.740088+0000","last_active":"2024-09-13T14:58:06.827426+0000","last_peered":"2024-09-13T14:58:06.827426+0000","last_clean":"2024-09-13T14:58:06.827426+0000","last_became_active":"2024-09-13T14:55:54.621088+0000","last_became_peered":"2024-09-13T14:55:54.621088+0000","last_unstale":"2024-09-13T14:58:06.827426+0000","last_undegraded":"2024-09-13T14:58:06.827426+0000","last_fullsized":"2024-09-13T14:58:06.827426+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.739931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.739931+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.959219+0000","scrub_duration":0.0093782959999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":810,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826175+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T14:58:07.826175+0000","last_peered":"2024-09-13T14:58:07.826175+0000","last_clean":"2024-09-13T14:58:07.826175+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T14:58:07.826175+0000","last_undegraded":"2024-09-13T14:58:07.826175+0000","last_fullsized":"2024-09-13T14:58:07.826175+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":900,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821693+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T14:58:06.821693+0000","last_peered":"2024-09-13T14:58:06.821693+0000","last_clean":"2024-09-13T14:58:06.821693+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T14:58:06.821693+0000","last_undegraded":"2024-09-13T14:58:06.821693+0000","last_fullsized":"2024-09-13T14:58:06.821693+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821694+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T14:58:07.821694+0000","last_peered":"2024-09-13T14:58:07.821694+0000","last_clean":"2024-09-13T14:58:07.821694+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T14:58:07.821694+0000","last_undegraded":"2024-09-13T14:58:07.821694+0000","last_fullsized":"2024-09-13T14:58:07.821694+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"484'15733","reported_seq":15776,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:18.141476+0000","last_change":"2024-09-13T14:54:28.816316+0000","last_active":"2024-09-13T14:58:18.141476+0000","last_peered":"2024-09-13T14:58:18.141476+0000","last_clean":"2024-09-13T14:58:18.141476+0000","last_became_active":"2024-09-13T14:54:28.816028+0000","last_became_peered":"2024-09-13T14:54:28.816028+0000","last_unstale":"2024-09-13T14:58:18.141476+0000","last_undegraded":"2024-09-13T14:58:18.141476+0000","last_fullsized":"2024-09-13T14:58:18.141476+0000","mapping_epoch":449,"log_start":"472'9556","ondisk_log_start":"472'9556","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6177,"log_dups_size":0,"ondisk_log_size":6177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:03.627959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14221312,"num_objects":217,"num_object_clones":0,"num_object_copies":651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15733,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":202,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822866+0000","last_change":"2024-09-13T14:56:00.807834+0000","last_active":"2024-09-13T14:58:06.822866+0000","last_peered":"2024-09-13T14:58:06.822866+0000","last_clean":"2024-09-13T14:58:06.822866+0000","last_became_active":"2024-09-13T14:55:54.609192+0000","last_became_peered":"2024-09-13T14:55:54.609192+0000","last_unstale":"2024-09-13T14:58:06.822866+0000","last_undegraded":"2024-09-13T14:58:06.822866+0000","last_fullsized":"2024-09-13T14:58:06.822866+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.807561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.807561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:44.901235+0000","scrub_duration":0.012160298999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":867,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826300+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T14:58:07.826300+0000","last_peered":"2024-09-13T14:58:07.826300+0000","last_clean":"2024-09-13T14:58:07.826300+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T14:58:07.826300+0000","last_undegraded":"2024-09-13T14:58:07.826300+0000","last_fullsized":"2024-09-13T14:58:07.826300+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1185,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826821+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T14:58:07.826821+0000","last_peered":"2024-09-13T14:58:07.826821+0000","last_clean":"2024-09-13T14:58:07.826821+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T14:58:07.826821+0000","last_undegraded":"2024-09-13T14:58:07.826821+0000","last_fullsized":"2024-09-13T14:58:07.826821+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":858,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.822660+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T14:58:07.822660+0000","last_peered":"2024-09-13T14:58:07.822660+0000","last_clean":"2024-09-13T14:58:07.822660+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T14:58:07.822660+0000","last_undegraded":"2024-09-13T14:58:07.822660+0000","last_fullsized":"2024-09-13T14:58:07.822660+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"484'15771","reported_seq":15814,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:17.948569+0000","last_change":"2024-09-13T14:54:28.839878+0000","last_active":"2024-09-13T14:58:17.948569+0000","last_peered":"2024-09-13T14:58:17.948569+0000","last_clean":"2024-09-13T14:58:17.948569+0000","last_became_active":"2024-09-13T14:54:28.839581+0000","last_became_peered":"2024-09-13T14:54:28.839581+0000","last_unstale":"2024-09-13T14:58:17.948569+0000","last_undegraded":"2024-09-13T14:58:17.948569+0000","last_fullsized":"2024-09-13T14:58:17.948569+0000","mapping_epoch":449,"log_start":"469'8784","ondisk_log_start":"469'8784","created":449,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:54:27.795864+0000","last_clean_scrub_stamp":"2024-09-13T14:54:27.795864+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:44.167354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13828096,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15771,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":718,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.824572+0000","last_change":"2024-09-13T14:56:00.000186+0000","last_active":"2024-09-13T14:58:07.824572+0000","last_peered":"2024-09-13T14:58:07.824572+0000","last_clean":"2024-09-13T14:58:07.824572+0000","last_became_active":"2024-09-13T14:55:54.611253+0000","last_became_peered":"2024-09-13T14:55:54.611253+0000","last_unstale":"2024-09-13T14:58:07.824572+0000","last_undegraded":"2024-09-13T14:58:07.824572+0000","last_fullsized":"2024-09-13T14:58:07.824572+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.999712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.999712+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:46.770712+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":148,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.830892+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T14:58:07.830892+0000","last_peered":"2024-09-13T14:58:07.830892+0000","last_clean":"2024-09-13T14:58:07.830892+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T14:58:07.830892+0000","last_undegraded":"2024-09-13T14:58:07.830892+0000","last_fullsized":"2024-09-13T14:58:07.830892+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":902,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.831317+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T14:58:06.831317+0000","last_peered":"2024-09-13T14:58:06.831317+0000","last_clean":"2024-09-13T14:58:06.831317+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T14:58:06.831317+0000","last_undegraded":"2024-09-13T14:58:06.831317+0000","last_fullsized":"2024-09-13T14:58:06.831317+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":196,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821732+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T14:58:06.821732+0000","last_peered":"2024-09-13T14:58:06.821732+0000","last_clean":"2024-09-13T14:58:06.821732+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T14:58:06.821732+0000","last_undegraded":"2024-09-13T14:58:06.821732+0000","last_fullsized":"2024-09-13T14:58:06.821732+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822428+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T14:58:06.822428+0000","last_peered":"2024-09-13T14:58:06.822428+0000","last_clean":"2024-09-13T14:58:06.822428+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T14:58:06.822428+0000","last_undegraded":"2024-09-13T14:58:06.822428+0000","last_fullsized":"2024-09-13T14:58:06.822428+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":192,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822570+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T14:58:06.822570+0000","last_peered":"2024-09-13T14:58:06.822570+0000","last_clean":"2024-09-13T14:58:06.822570+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T14:58:06.822570+0000","last_undegraded":"2024-09-13T14:58:06.822570+0000","last_fullsized":"2024-09-13T14:58:06.822570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":797,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.827941+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T14:58:07.827941+0000","last_peered":"2024-09-13T14:58:07.827941+0000","last_clean":"2024-09-13T14:58:07.827941+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T14:58:07.827941+0000","last_undegraded":"2024-09-13T14:58:07.827941+0000","last_fullsized":"2024-09-13T14:58:07.827941+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":483,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821364+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T14:58:06.821364+0000","last_peered":"2024-09-13T14:58:06.821364+0000","last_clean":"2024-09-13T14:58:06.821364+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T14:58:06.821364+0000","last_undegraded":"2024-09-13T14:58:06.821364+0000","last_fullsized":"2024-09-13T14:58:06.821364+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":282,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:08.732601+0000","last_change":"2024-09-13T14:58:08.732515+0000","last_active":"2024-09-13T14:58:08.732601+0000","last_peered":"2024-09-13T14:58:08.732601+0000","last_clean":"2024-09-13T14:58:08.732601+0000","last_became_active":"2024-09-13T14:58:07.825813+0000","last_became_peered":"2024-09-13T14:58:07.825813+0000","last_unstale":"2024-09-13T14:58:08.732601+0000","last_undegraded":"2024-09-13T14:58:08.732601+0000","last_fullsized":"2024-09-13T14:58:08.732601+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:08.732438+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.732438+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:14.508509+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":449,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822987+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T14:58:06.822987+0000","last_peered":"2024-09-13T14:58:06.822987+0000","last_clean":"2024-09-13T14:58:06.822987+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T14:58:06.822987+0000","last_undegraded":"2024-09-13T14:58:06.822987+0000","last_fullsized":"2024-09-13T14:58:06.822987+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":801,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.823003+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T14:58:07.823003+0000","last_peered":"2024-09-13T14:58:07.823003+0000","last_clean":"2024-09-13T14:58:07.823003+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T14:58:07.823003+0000","last_undegraded":"2024-09-13T14:58:07.823003+0000","last_fullsized":"2024-09-13T14:58:07.823003+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.827612+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T14:58:07.827612+0000","last_peered":"2024-09-13T14:58:07.827612+0000","last_clean":"2024-09-13T14:58:07.827612+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T14:58:07.827612+0000","last_undegraded":"2024-09-13T14:58:07.827612+0000","last_fullsized":"2024-09-13T14:58:07.827612+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":857,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.822575+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T14:58:07.822575+0000","last_peered":"2024-09-13T14:58:07.822575+0000","last_clean":"2024-09-13T14:58:07.822575+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T14:58:07.822575+0000","last_undegraded":"2024-09-13T14:58:07.822575+0000","last_fullsized":"2024-09-13T14:58:07.822575+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":481,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.831179+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T14:58:06.831179+0000","last_peered":"2024-09-13T14:58:06.831179+0000","last_clean":"2024-09-13T14:58:06.831179+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T14:58:06.831179+0000","last_undegraded":"2024-09-13T14:58:06.831179+0000","last_fullsized":"2024-09-13T14:58:06.831179+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":889,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.822737+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T14:58:07.822737+0000","last_peered":"2024-09-13T14:58:07.822737+0000","last_clean":"2024-09-13T14:58:07.822737+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T14:58:07.822737+0000","last_undegraded":"2024-09-13T14:58:07.822737+0000","last_fullsized":"2024-09-13T14:58:07.822737+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.831062+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T14:58:07.831062+0000","last_peered":"2024-09-13T14:58:07.831062+0000","last_clean":"2024-09-13T14:58:07.831062+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T14:58:07.831062+0000","last_undegraded":"2024-09-13T14:58:07.831062+0000","last_fullsized":"2024-09-13T14:58:07.831062+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":821,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.827252+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T14:58:07.827252+0000","last_peered":"2024-09-13T14:58:07.827252+0000","last_clean":"2024-09-13T14:58:07.827252+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T14:58:07.827252+0000","last_undegraded":"2024-09-13T14:58:07.827252+0000","last_fullsized":"2024-09-13T14:58:07.827252+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":870,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.840262+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T14:58:07.840262+0000","last_peered":"2024-09-13T14:58:07.840262+0000","last_clean":"2024-09-13T14:58:07.840262+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T14:58:07.840262+0000","last_undegraded":"2024-09-13T14:58:07.840262+0000","last_fullsized":"2024-09-13T14:58:07.840262+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":181,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822148+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T14:58:06.822148+0000","last_peered":"2024-09-13T14:58:06.822148+0000","last_clean":"2024-09-13T14:58:06.822148+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T14:58:06.822148+0000","last_undegraded":"2024-09-13T14:58:06.822148+0000","last_fullsized":"2024-09-13T14:58:06.822148+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":767,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.826410+0000","last_change":"2024-09-13T14:39:23.287128+0000","last_active":"2024-09-13T14:58:07.826410+0000","last_peered":"2024-09-13T14:58:07.826410+0000","last_clean":"2024-09-13T14:58:07.826410+0000","last_became_active":"2024-09-13T14:39:23.286352+0000","last_became_peered":"2024-09-13T14:39:23.286352+0000","last_unstale":"2024-09-13T14:58:07.826410+0000","last_undegraded":"2024-09-13T14:58:07.826410+0000","last_fullsized":"2024-09-13T14:58:07.826410+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:13.443347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":144,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.821819+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T14:58:07.821819+0000","last_peered":"2024-09-13T14:58:07.821819+0000","last_clean":"2024-09-13T14:58:07.821819+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T14:58:07.821819+0000","last_undegraded":"2024-09-13T14:58:07.821819+0000","last_fullsized":"2024-09-13T14:58:07.821819+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822354+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T14:58:06.822354+0000","last_peered":"2024-09-13T14:58:06.822354+0000","last_clean":"2024-09-13T14:58:06.822354+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T14:58:06.822354+0000","last_undegraded":"2024-09-13T14:58:06.822354+0000","last_fullsized":"2024-09-13T14:58:06.822354+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":449,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.818631+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T14:58:06.818631+0000","last_peered":"2024-09-13T14:58:06.818631+0000","last_clean":"2024-09-13T14:58:06.818631+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T14:58:06.818631+0000","last_undegraded":"2024-09-13T14:58:06.818631+0000","last_fullsized":"2024-09-13T14:58:06.818631+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":201,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.822119+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T14:58:06.822119+0000","last_peered":"2024-09-13T14:58:06.822119+0000","last_clean":"2024-09-13T14:58:06.822119+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T14:58:06.822119+0000","last_undegraded":"2024-09-13T14:58:06.822119+0000","last_fullsized":"2024-09-13T14:58:06.822119+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":415,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.819020+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T14:58:06.819020+0000","last_peered":"2024-09-13T14:58:06.819020+0000","last_clean":"2024-09-13T14:58:06.819020+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T14:58:06.819020+0000","last_undegraded":"2024-09-13T14:58:06.819020+0000","last_fullsized":"2024-09-13T14:58:06.819020+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":881,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.823366+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T14:58:07.823366+0000","last_peered":"2024-09-13T14:58:07.823366+0000","last_clean":"2024-09-13T14:58:07.823366+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T14:58:07.823366+0000","last_undegraded":"2024-09-13T14:58:07.823366+0000","last_fullsized":"2024-09-13T14:58:07.823366+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":855,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.823850+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T14:58:07.823850+0000","last_peered":"2024-09-13T14:58:07.823850+0000","last_clean":"2024-09-13T14:58:07.823850+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T14:58:07.823850+0000","last_undegraded":"2024-09-13T14:58:07.823850+0000","last_fullsized":"2024-09-13T14:58:07.823850+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":182,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-09-13T14:58:06.821079+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T14:58:06.821079+0000","last_peered":"2024-09-13T14:58:06.821079+0000","last_clean":"2024-09-13T14:58:06.821079+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T14:58:06.821079+0000","last_undegraded":"2024-09-13T14:58:06.821079+0000","last_fullsized":"2024-09-13T14:58:06.821079+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"64'4","reported_seq":150,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-09-13T14:58:07.827509+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T14:58:07.827509+0000","last_peered":"2024-09-13T14:58:07.827509+0000","last_clean":"2024-09-13T14:58:07.827509+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T14:58:07.827509+0000","last_undegraded":"2024-09-13T14:58:07.827509+0000","last_fullsized":"2024-09-13T14:58:07.827509+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":235798552,"num_objects":3599,"num_object_clones":0,"num_object_copies":10797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3599,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":249005,"num_write_kb":8083265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":748761088,"data_stored":748748872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116871,"ondisk_log_size":116871,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398066,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972580,"kb_used_data":76572,"kb_used_omap":1,"kb_used_meta":895998,"kb_avail":92772572,"statfs":{"total":95995035648,"available":94999113728,"internally_reserved":0,"allocated":78409728,"data_stored":75867690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917502296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594294,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994328,"kb_used_data":104016,"kb_used_omap":1,"kb_used_meta":890302,"kb_avail":92750824,"statfs":{"total":95995035648,"available":94976843776,"internally_reserved":0,"allocated":106512384,"data_stored":103981956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911669592},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665748,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005392,"kb_used_data":75208,"kb_used_omap":9,"kb_used_meta":930166,"kb_avail":92739760,"statfs":{"total":95995035648,"available":94965514240,"internally_reserved":0,"allocated":77012992,"data_stored":74487039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":952490184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986750,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027556,"kb_used_data":131868,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92717596,"statfs":{"total":95995035648,"available":94942818304,"internally_reserved":0,"allocated":135032832,"data_stored":132490748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697310,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183376,"kb_used_data":66120,"kb_used_omap":4,"kb_used_meta":1117243,"kb_avail":92561776,"statfs":{"total":95995035648,"available":94783258624,"internally_reserved":0,"allocated":67706880,"data_stored":65181008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1144056907},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860836,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1034836,"kb_used_data":141900,"kb_used_omap":8,"kb_used_meta":892855,"kb_avail":92710316,"statfs":{"total":95995035648,"available":94935363584,"internally_reserved":0,"allocated":145305600,"data_stored":142776146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":914284364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020507,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":739148,"kb_used_data":83140,"kb_used_omap":9,"kb_used_meta":655926,"kb_avail":93006004,"statfs":{"total":95995035648,"available":95238148096,"internally_reserved":0,"allocated":85135360,"data_stored":82612961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":671669061},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187898,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1219216,"kb_used_data":181256,"kb_used_omap":7,"kb_used_meta":1037880,"kb_avail":92525936,"statfs":{"total":95995035648,"available":94746558464,"internally_reserved":0,"allocated":185606144,"data_stored":183079939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":1062789451},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":156569600,"data_stored":156565528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":68943872,"data_stored":68943872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":136642560,"data_stored":136642560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":58982400,"data_stored":58982400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":111415296,"data_stored":111411224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":65011712,"data_stored":65011712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":97849344,"data_stored":97845272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":53346304,"data_stored":53346304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:58:20.565 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:58:20.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-09-13T14:58:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:20.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:20.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:20.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:20.839 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.869 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.971 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:21.003 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:21.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:21.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:21.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:21.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:21.154 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:20 smithi017 ceph-mon[103001]: from='client.37323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:21.154 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:20 smithi017 ceph-mon[103001]: pgmap v1610: 109 pgs: 109 active+clean; 229 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:21.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:20 smithi017 ceph-mon[107556]: from='client.37323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:21.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:20 smithi017 ceph-mon[107556]: pgmap v1610: 109 pgs: 109 active+clean; 229 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:21.243 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:20 smithi189 ceph-mon[90992]: from='client.37323 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:58:21.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:20 smithi189 ceph-mon[90992]: pgmap v1610: 109 pgs: 109 active+clean; 229 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:58:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.279 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:21.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:21.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:21.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:21.514 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:21.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:21.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 126301 objects 2024-09-13T14:58:21.798 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :128.783 2024-09-13T14:58:21.818 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:58:21.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:21.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:22.001 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:22.044 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.066 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:22.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:22.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:22.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:22.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:58:22.726 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:22.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:58:22.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:22.743 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:58:22.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:22.760 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:58:22.760 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:22.777 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:58:22.777 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:22.793 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:58:22.793 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:22.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:22.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:58:22.809 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:22.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:22.825 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:58:22.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:22.923 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:23.047 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.126 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:22 smithi189 ceph-mon[90992]: pgmap v1611: 109 pgs: 109 active+clean; 211 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:58:23.126 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2730270974' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:23.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:23.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:22 smithi017 ceph-mon[103001]: pgmap v1611: 109 pgs: 109 active+clean; 211 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:58:23.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2730270974' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:23.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:22 smithi017 ceph-mon[107556]: pgmap v1611: 109 pgs: 109 active+clean; 211 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:58:23.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2730270974' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:23.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:23.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.320 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.377 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:23.377 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:23.422 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.444 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:23.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.546 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.568 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.670 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.692 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:23.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:23.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:23.871 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:23.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:23.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:24.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:24.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:24.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.303 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:58:24.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:24.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:24.542 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:24.552 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:24.563 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:24.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:24.665 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:24.687 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:24.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.913 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.937 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:24.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:24.991 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:24 smithi017 ceph-mon[103001]: pgmap v1612: 109 pgs: 109 active+clean; 45 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T14:58:24.991 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:24 smithi017 ceph-mon[107556]: pgmap v1612: 109 pgs: 109 active+clean; 45 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T14:58:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:24 smithi189 ceph-mon[90992]: pgmap v1612: 109 pgs: 109 active+clean; 45 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-13T14:58:25.038 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:25.059 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:25.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:25.558 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:25.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:25.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:25.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:25.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:25.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:25.885 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 31 2024-09-13T14:58:25.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:25.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:26.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:26.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:26.123 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1924136129' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.124 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.159 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.180 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1924136129' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1924136129' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-13T14:58:26.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:26.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:26.363 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:26.363 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:26.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.427 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.529 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:26.551 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:26.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:26.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.780 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.801 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:26.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.884 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-13T14:58:26.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-13T14:58:26.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.954 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:26 smithi189 ceph-mon[90992]: pgmap v1613: 109 pgs: 109 active+clean; 7.8 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:58:26.954 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:58:26.954 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:26 smithi189 ceph-mon[90992]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:58:26.977 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:58:27.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.074 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[103001]: pgmap v1613: 109 pgs: 109 active+clean; 7.8 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[103001]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[107556]: pgmap v1613: 109 pgs: 109 active+clean; 7.8 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-13T14:58:27.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:26 smithi017 ceph-mon[107556]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:58:27.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:27.199 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:27.226 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:27.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.323 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:27.346 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:27.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:27.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:27] "GET /metrics HTTP/1.1" 200 36707 "" "Prometheus/2.51.0" 2024-09-13T14:58:27.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:27.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:27.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:27.697 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:27.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:27.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:28.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.071 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:28.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:28.150 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-13T14:58:28.172 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.193 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:28 smithi189 ceph-mon[90992]: pgmap v1615: 109 pgs: 109 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 417 op/s 2024-09-13T14:58:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:28 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:28.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:28.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:28.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:28.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[103001]: pgmap v1615: 109 pgs: 109 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 417 op/s 2024-09-13T14:58:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[107556]: pgmap v1615: 109 pgs: 109 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 417 op/s 2024-09-13T14:58:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:28.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:28.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:28.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:28.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:28.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:28.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.829 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:28.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.931 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.953 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.973 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:28.973 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:29.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:29.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:29.185 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.208 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:29.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:29 smithi189 ceph-mon[90992]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:29 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:58:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:29.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:29 smithi189 ceph-mon[90992]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:58:29.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:29.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:29.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:29.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[103001]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:29.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:58:29.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:29.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[103001]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:58:29.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[107556]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:29.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-13T14:58:29.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3520038562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:29.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:29 smithi017 ceph-mon[107556]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:58:29.438 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:29.459 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:29.561 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.582 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.619 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:29.654 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:58:29.684 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:29.718 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:29.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:29.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:29.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:29.900 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:29.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:30.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:30.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:30.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:30.347 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:30.347 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:30 smithi189 ceph-mon[90992]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:58:30.347 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:30 smithi189 ceph-mon[90992]: pgmap v1618: 119 pgs: 7 peering, 10 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 107 op/s 2024-09-13T14:58:30.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:30.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:30.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:30 smithi017 ceph-mon[103001]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:58:30.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:30 smithi017 ceph-mon[103001]: pgmap v1618: 119 pgs: 7 peering, 10 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 107 op/s 2024-09-13T14:58:30.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:30 smithi017 ceph-mon[107556]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:58:30.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:30 smithi017 ceph-mon[107556]: pgmap v1618: 119 pgs: 7 peering, 10 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 107 op/s 2024-09-13T14:58:30.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.601 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:30.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.780 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:58:30.780 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:30.796 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:58:30.797 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:30.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:58:30.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:30.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.829 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:58:30.829 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:30.845 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:58:30.845 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:30.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.861 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:58:30.861 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:30.876 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:58:30.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:30.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:58:30.893 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:30.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.974 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:30.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:31.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:31.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:31.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:31.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.346 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:31.448 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[103001]: 1.b scrub starts 2024-09-13T14:58:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[103001]: 1.c scrub starts 2024-09-13T14:58:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2697585059' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[107556]: 1.b scrub starts 2024-09-13T14:58:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[107556]: 1.c scrub starts 2024-09-13T14:58:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2697585059' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:31.469 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:31 smithi189 ceph-mon[90992]: 1.b scrub starts 2024-09-13T14:58:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:31 smithi189 ceph-mon[90992]: 1.c scrub starts 2024-09-13T14:58:31.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2697585059' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:31.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:31.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:31.697 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.719 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:31.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:31.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:31.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:31.885 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:58:31.885 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:31.888 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-13T14:58:31.888 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:58:31.888 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:58:31.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-13T14:58:31.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:31.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:32.070 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.093 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:32.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:32.156 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:32.156 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:58:32.156 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:58:32.156 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:58:32.156 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:32.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:32.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:32.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.343 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:32.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:32.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[103001]: 1.1b scrub starts 2024-09-13T14:58:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[103001]: 1.1c scrub starts 2024-09-13T14:58:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[103001]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 7 peering, 9 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[107556]: 1.1b scrub starts 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[107556]: 1.1c scrub starts 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:32 smithi017 ceph-mon[107556]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 7 peering, 9 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:32 smithi189 ceph-mon[90992]: 1.1b scrub starts 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:32 smithi189 ceph-mon[90992]: 1.1c scrub starts 2024-09-13T14:58:32.451 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:32 smithi189 ceph-mon[90992]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 7 peering, 9 unknown, 102 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2024-09-13T14:58:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:32.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:32.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:32.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:58:32.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:32.783 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:58:32.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:32.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:32.897 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:32.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:32.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:33.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:33.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:33.164 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:33.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:33.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:33.324 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.347 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:33.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:33.361 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' removed 2024-09-13T14:58:33.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:33.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:33.573 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.595 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:33.615 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:33.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:33 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/283983490' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:33 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:33 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/283983490' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:33 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:33.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:33.736 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:33 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/283983490' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.736 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:33 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:33.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:33.947 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.970 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:33.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:34.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:34.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:34.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:34.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:34.320 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.342 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:34.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.557 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:34 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:34.557 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:34 smithi189 ceph-mon[90992]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:58:34.557 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:34 smithi189 ceph-mon[90992]: pgmap v1621: 103 pgs: 1 active+clean+scrubbing, 3 peering, 3 unknown, 96 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:34.570 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.579 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:58:34.580 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-13T14:58:34.580 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T14:58:34.580 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T14:58:34.580 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:58:34.581 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:58:34.593 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.597 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:58:34.597 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:58:34.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:34.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:34.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[107556]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:58:34.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[107556]: pgmap v1621: 103 pgs: 1 active+clean+scrubbing, 3 peering, 3 unknown, 96 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:34.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:34.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[103001]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:58:34.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:34 smithi017 ceph-mon[103001]: pgmap v1621: 103 pgs: 1 active+clean+scrubbing, 3 peering, 3 unknown, 96 active+clean; 4.4 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:34.687 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:58:34.687 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:58:34.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:34.744 INFO:tasks.workunit:timeout=3h 2024-09-13T14:58:34.745 INFO:tasks.workunit:cleanup=True 2024-09-13T14:58:34.745 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:58:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:34.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:34.776 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:58:34.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:34.862 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:34.862 INFO:teuthology.orchestra.run.smithi189.stdout: "success": "" 2024-09-13T14:58:34.862 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:34.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:34.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:35.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:35.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:35.108 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.131 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:35.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:35.232 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.255 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:35.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:35.259 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:35.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:35.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:35 smithi017 ceph-mon[103001]: 1.c scrub starts 2024-09-13T14:58:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:35 smithi017 ceph-mon[107556]: 1.c scrub starts 2024-09-13T14:58:35.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:35.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:35 smithi189 ceph-mon[90992]: 1.c scrub starts 2024-09-13T14:58:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.608 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.629 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:35.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:35.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:35.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.855 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.877 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:35.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:35.980 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.002 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:36.228 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.251 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:36.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:36.383 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:58:36.484 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: 1.b scrub starts 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: pgmap v1622: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: 1.1c scrub starts 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2319054397' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:36 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.507 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:36.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:36.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:36.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: 1.b scrub starts 2024-09-13T14:58:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: pgmap v1622: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: 1.1c scrub starts 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2319054397' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: 1.b scrub starts 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: pgmap v1622: 103 pgs: 3 active+clean+scrubbing, 100 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: 1.1c scrub starts 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2319054397' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:36 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:36.733 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.736 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:91008 10.7 10:eec1ab19:::benchmark_data_smithi017_372078_object91007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:37.689529+0000", 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 179.041529774, 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21906622000000001, 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.737 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 91008 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689529+0000", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689529+0000", 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.738 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689528+0000", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689582+0000", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4345000000000002e-05 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689591+0000", 2024-09-13T14:58:36.739 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3159999999999997e-06 2024-09-13T14:58:36.740 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.740 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.740 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.741 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.689595+0000", 2024-09-13T14:58:36.741 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7759999999999997e-06 2024-09-13T14:58:36.741 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.741 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.741 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.695483+0000", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058871419999999997 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.695589+0000", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010658900000000001 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.695702+0000", 2024-09-13T14:58:36.742 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011289399999999999 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.703416+0000", 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077142449999999998 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.706440+0000", 2024-09-13T14:58:36.743 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0030241589999999998 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.908498+0000", 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.202057711 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.908549+0000", 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0713e-05 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.744 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.908595+0000", 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6060000000000003e-05 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:91016 10.7 10:eaf86f8f:::benchmark_data_smithi017_372078_object91015:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:37.697042+0000", 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 179.03401662600001, 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.22260943799999999, 2024-09-13T14:58:36.745 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 91016 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697042+0000", 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.746 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697042+0000", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697040+0000", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697091+0000", 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0133999999999998e-05 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.747 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697104+0000", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3867e-05 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.697110+0000", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7270000000000004e-06 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.700969+0000", 2024-09-13T14:58:36.748 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038590870000000002 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701066+0000", 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6976000000000001e-05 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701177+0000", 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011034100000000001 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.749 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.705061+0000", 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003883985 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.706522+0000", 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0014615940000000001 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.919559+0000", 2024-09-13T14:58:36.750 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21303715200000001 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.919613+0000", 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3909999999999997e-05 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.919651+0000", 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8046000000000001e-05 2024-09-13T14:58:36.751 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:91023 10.a 10:553308fa:::benchmark_data_smithi017_372078_object91022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:37.701119+0000", 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 179.029939055, 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21612595600000001, 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.752 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 91023 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701119+0000", 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701119+0000", 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.753 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701118+0000", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701267+0000", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014881599999999999 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701280+0000", 2024-09-13T14:58:36.754 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3342e-05 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701287+0000", 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3210000000000001e-06 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701327+0000", 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0405999999999999e-05 2024-09-13T14:58:36.755 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701414+0000", 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7168000000000003e-05 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.701501+0000", 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7239999999999998e-05 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.756 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.703808+0000", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0023068020000000002 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.705543+0000", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0017345450000000001 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.917160+0000", 2024-09-13T14:58:36.757 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.2116171 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.917208+0000", 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8523000000000003e-05 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:37.917245+0000", 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7052999999999999e-05 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.758 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111439 10.f 10:ffe00de6:::benchmark_data_smithi017_372078_object111438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:56.945324+0000", 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 159.785734225, 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.276318644, 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.759 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111439 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.760 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.761 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945324+0000", 2024-09-13T14:58:36.761 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.761 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.761 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.761 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945324+0000", 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945323+0000", 2024-09-13T14:58:36.762 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945360+0000", 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7098999999999998e-05 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.763 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945371+0000", 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1497e-05 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945377+0000", 2024-09-13T14:58:36.764 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3399999999999997e-06 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.949213+0000", 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003836108 2024-09-13T14:58:36.765 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.949321+0000", 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010802900000000001 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.766 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.949470+0000", 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000149097 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.961653+0000", 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012182876 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969537+0000", 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078837239999999999 2024-09-13T14:58:36.768 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221558+0000", 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25202135399999998 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221603+0000", 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5345e-05 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.769 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221643+0000", 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9637e-05 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111444 10.f 10:fe4aa480:::benchmark_data_smithi017_372078_object111443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:56.945830+0000", 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 159.78522871499999, 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.27601351200000002, 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.770 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111444 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945830+0000", 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.771 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945830+0000", 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945829+0000", 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945875+0000", 2024-09-13T14:58:36.772 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6171999999999998e-05 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945885+0000", 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.944e-06 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.945890+0000", 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1220000000000003e-06 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.773 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.952789+0000", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068984390000000001 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.952934+0000", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014549800000000001 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953057+0000", 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123056 2024-09-13T14:58:36.774 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.961847+0000", 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0087899380000000006 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.976491+0000", 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014644079000000001 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.775 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221722+0000", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.245231158 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221789+0000", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6753000000000003e-05 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221843+0000", 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4160999999999999e-05 2024-09-13T14:58:36.776 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111447 10.f 10:f081956e:::benchmark_data_smithi017_372078_object111446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:56.953338+0000", 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 159.77772026700001, 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.26911028799999998, 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.777 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111447 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953338+0000", 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.778 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953338+0000", 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953336+0000", 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953371+0000", 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5200000000000002e-05 2024-09-13T14:58:36.779 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953384+0000", 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2184000000000001e-05 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.953389+0000", 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3669999999999996e-06 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.780 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.955923+0000", 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002534026 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956027+0000", 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000103675 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.781 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956127+0000", 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9795000000000003e-05 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.961854+0000", 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0057271400000000004 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.782 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.976559+0000", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014705441 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221893+0000", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24533384499999999 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.221925+0000", 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1572999999999997e-05 2024-09-13T14:58:36.783 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222448+0000", 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00052395399999999998 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.784 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111451 10.f 10:f0226e31:::benchmark_data_smithi017_372078_object111450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:56.956795+0000", 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 159.774263765, 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.26579822800000003, 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.785 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111451 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956795+0000", 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.786 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956795+0000", 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956793+0000", 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956853+0000", 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0003000000000001e-05 2024-09-13T14:58:36.787 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956868+0000", 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5183e-05 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.956874+0000", 2024-09-13T14:58:36.788 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7010000000000001e-06 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.958991+0000", 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0021163979999999998 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.959075+0000", 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4025999999999998e-05 2024-09-13T14:58:36.790 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.959164+0000", 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9499999999999994e-05 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.961860+0000", 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0026959430000000001 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.791 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.976719+0000", 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.014859277000000001 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222512+0000", 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24579300200000001 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.792 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222556+0000", 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4014000000000002e-05 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222593+0000", 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6613000000000003e-05 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.793 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111460 10.f 10:f78880b5:::benchmark_data_smithi017_372078_object111459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:56.969332+0000", 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 159.761726557, 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25344129700000001, 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.794 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111460 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969332+0000", 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.795 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969332+0000", 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969331+0000", 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969387+0000", 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6366999999999998e-05 2024-09-13T14:58:36.796 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969398+0000", 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0458999999999999e-05 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969403+0000", 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9279999999999998e-06 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.797 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969548+0000", 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014437799999999999 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969634+0000", 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6599000000000002e-05 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.798 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.969746+0000", 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011142 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.972462+0000", 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027163030000000002 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:56.976806+0000", 2024-09-13T14:58:36.799 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0043443199999999996 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222694+0000", 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.245887418 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.800 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222738+0000", 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4397000000000002e-05 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.222773+0000", 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4922000000000002e-05 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.801 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:111994 10.f 10:fb8c6cd5:::benchmark_data_smithi017_372078_object111993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:57.776195+0000", 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 158.95486298700001, 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.223034765, 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.802 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 111994 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776195+0000", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776195+0000", 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.803 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776194+0000", 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.804 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776246+0000", 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1641000000000002e-05 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776257+0000", 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0998999999999999e-05 2024-09-13T14:58:36.805 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.806 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.806 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776261+0000", 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1960000000000001e-06 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776300+0000", 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9112000000000001e-05 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.808 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776395+0000", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4660000000000005e-05 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.776489+0000", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4369000000000004e-05 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784071+0000", 2024-09-13T14:58:36.809 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075822429999999998 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.800933+0000", 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016862073000000002 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999165+0000", 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.198231981 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.810 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999206+0000", 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0518000000000001e-05 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999230+0000", 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4365000000000001e-05 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.811 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:112003 10.f 10:f5773472:::benchmark_data_smithi017_372078_object112002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:57.780257+0000", 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 158.95080140300001, 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21906556399999999, 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.812 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 112003 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780257+0000", 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.813 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780257+0000", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780256+0000", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780287+0000", 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0874999999999998e-05 2024-09-13T14:58:36.814 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.815 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.815 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.815 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780296+0000", 2024-09-13T14:58:36.815 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.5710000000000004e-06 2024-09-13T14:58:36.815 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780301+0000", 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.074e-06 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.816 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780357+0000", 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5353999999999999e-05 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.817 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.818 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780468+0000", 2024-09-13T14:58:36.818 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111797 2024-09-13T14:58:36.818 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.818 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.818 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.780596+0000", 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012750899999999999 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.819 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784219+0000", 2024-09-13T14:58:36.820 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003623037 2024-09-13T14:58:36.820 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.820 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.820 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.820 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.800970+0000", 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016750661999999999 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999267+0000", 2024-09-13T14:58:36.821 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.19829739099999999 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999300+0000", 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3034000000000003e-05 2024-09-13T14:58:36.822 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999323+0000", 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2600999999999999e-05 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.823 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.824 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.824 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.824 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.824 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:112007 10.f 10:f6c364c4:::benchmark_data_smithi017_372078_object112006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.824 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:57.784285+0000", 2024-09-13T14:58:36.825 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 158.946773783, 2024-09-13T14:58:36.825 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21511233099999999, 2024-09-13T14:58:36.825 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 112007 2024-09-13T14:58:36.826 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784285+0000", 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.827 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784285+0000", 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784284+0000", 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.829 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784321+0000", 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7777000000000001e-05 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.830 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.831 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784333+0000", 2024-09-13T14:58:36.831 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1812e-05 2024-09-13T14:58:36.831 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.831 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.831 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784339+0000", 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0800000000000002e-06 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.832 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784401+0000", 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2106000000000005e-05 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784509+0000", 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000107348 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784652+0000", 2024-09-13T14:58:36.833 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014320000000000001 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789478+0000", 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048256779999999999 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.808802+0000", 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019324258 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.834 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999356+0000", 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.190554165 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999379+0000", 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2900000000000001e-05 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999397+0000", 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7988000000000001e-05 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.835 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:112008 10.7 10:e830d5a5:::benchmark_data_smithi017_372078_object112007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:57.784621+0000", 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 158.94643757099999, 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21477133500000001, 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.836 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 112008 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784621+0000", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784621+0000", 2024-09-13T14:58:36.837 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784620+0000", 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784657+0000", 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7206000000000003e-05 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.838 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784669+0000", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1932e-05 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784675+0000", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9190000000000004e-06 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784717+0000", 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2080999999999999e-05 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.839 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784832+0000", 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000114589 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.784967+0000", 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013514500000000001 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.840 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789027+0000", 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00406034 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789190+0000", 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000162895 2024-09-13T14:58:36.841 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999350+0000", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21015953200000001 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999373+0000", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3782999999999999e-05 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.999392+0000", 2024-09-13T14:58:36.842 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8851999999999998e-05 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:112010 10.f 10:fdd04ca2:::benchmark_data_smithi017_372078_object112009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:55:57.789809+0000", 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 158.94124900400001, 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.22150889200000001, 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.843 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 112010 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789809+0000", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789809+0000", 2024-09-13T14:58:36.844 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789809+0000", 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789879+0000", 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0222000000000006e-05 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.845 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789891+0000", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1875999999999999e-05 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789896+0000", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6489999999999996e-06 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.789938+0000", 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1284e-05 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.846 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.790059+0000", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000120864 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.790218+0000", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015907200000000001 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.793175+0000", 2024-09-13T14:58:36.847 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0029571179999999999 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:57.808954+0000", 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015779652000000002 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:58.011217+0000", 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.202263051 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.848 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:58.011278+0000", 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0600000000000003e-05 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:55:58.011318+0000", 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0314000000000001e-05 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.849 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.850 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:36.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.851 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123196 10.f 10:fe61610c:::benchmark_data_smithi017_372078_object123195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.851 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.563195+0000", 2024-09-13T14:58:36.851 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.167863947, 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.26777970899999998, 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123196 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563195+0000", 2024-09-13T14:58:36.852 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563195+0000", 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563188+0000", 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999938 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.853 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563269+0000", 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1151999999999998e-05 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563287+0000", 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7289999999999999e-05 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563292+0000", 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.378e-06 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.854 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563633+0000", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034156099999999999 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563835+0000", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000201335 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563961+0000", 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000126291 2024-09-13T14:58:36.855 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.572245+0000", 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0082842369999999999 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.578358+0000", 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061127320000000001 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.856 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.830872+0000", 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.25251369200000001 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.830928+0000", 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6388999999999998e-05 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.830974+0000", 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6088999999999999e-05 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.857 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123198 10.f 10:f78e9f7e:::benchmark_data_smithi017_372078_object123197:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.563374+0000", 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.16768495299999, 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.27685379500000001, 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.858 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123198 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563374+0000", 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563374+0000", 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.859 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563373+0000", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563428+0000", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4559999999999999e-05 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563434+0000", 2024-09-13T14:58:36.860 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8889999999999996e-06 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.563438+0000", 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8659999999999999e-06 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.568031+0000", 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045923839999999997 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.861 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.568384+0000", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00035298399999999998 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.568473+0000", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9765999999999994e-05 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.573050+0000", 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045767669999999998 2024-09-13T14:58:36.862 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.578865+0000", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0058151119999999999 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840139+0000", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.261273439 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840193+0000", 2024-09-13T14:58:36.863 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3857000000000003e-05 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840227+0000", 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4652e-05 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123221 10.f 10:fafbff9e:::benchmark_data_smithi017_372078_object123220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.864 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.592443+0000", 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.13861527099999, 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24833656400000001, 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123221 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.865 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592443+0000", 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592443+0000", 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592442+0000", 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592481+0000", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9916000000000002e-05 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592499+0000", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8031999999999999e-05 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592506+0000", 2024-09-13T14:58:36.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592548+0000", 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1922000000000003e-05 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592662+0000", 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011450700000000001 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.592785+0000", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012293199999999999 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.596998+0000", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0042126220000000001 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.597151+0000", 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000153358 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840699+0000", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24354821600000001 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840746+0000", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6400999999999998e-05 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.840780+0000", 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4209999999999999e-05 2024-09-13T14:58:36.870 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123243 10.f 10:fbd924f6:::benchmark_data_smithi017_372078_object123242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.615179+0000", 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.11587984799999, 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.23139541599999999, 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.871 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123243 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615179+0000", 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615179+0000", 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.872 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615177+0000", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615220+0000", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2657000000000002e-05 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615236+0000", 2024-09-13T14:58:36.873 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6342e-05 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615243+0000", 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5239999999999997e-06 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615695+0000", 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000452046 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.874 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615809+0000", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0001147 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.615941+0000", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013184000000000001 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.619844+0000", 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039028119999999999 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.626010+0000", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061657530000000004 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846498+0000", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.220488145 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846540+0000", 2024-09-13T14:58:36.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2647e-05 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846574+0000", 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3547999999999998e-05 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123248 10.f 10:f6f9b0d7:::benchmark_data_smithi017_372078_object123247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.877 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.621208+0000", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.10985012, 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.225563443, 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123248 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621208+0000", 2024-09-13T14:58:36.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621208+0000", 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621208+0000", 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621272+0000", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3932999999999999e-05 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621288+0000", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6269e-05 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621293+0000", 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.507e-06 2024-09-13T14:58:36.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621342+0000", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8643999999999997e-05 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621524+0000", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000182253 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.621695+0000", 2024-09-13T14:58:36.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000170805 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.625380+0000", 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0036853179999999999 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.626900+0000", 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001519451 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846669+0000", 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21976941899999999 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846730+0000", 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0361e-05 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846772+0000", 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2064999999999998e-05 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.883 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:123250 10.7 10:e99e24ea:::benchmark_data_smithi017_372078_object123249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:56:09.627479+0000", 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 147.10357905000001, 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.21915014999999999, 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.884 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 123250 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627479+0000", 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627479+0000", 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627478+0000", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627501+0000", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2294999999999999e-05 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627513+0000", 2024-09-13T14:58:36.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2075999999999999e-05 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627519+0000", 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0260000000000004e-06 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627562+0000", 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3687999999999999e-05 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.887 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627691+0000", 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012906300000000001 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 3,5", 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.627844+0000", 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00015255600000000001 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.631940+0000", 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0040962530000000002 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.632159+0000", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021853199999999999 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846543+0000", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.214383818 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846590+0000", 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7511e-05 2024-09-13T14:58:36.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:56:09.846630+0000", 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9409000000000002e-05 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.37095.0:227042 10.f 10:fac4d207:::benchmark_data_smithi017_372078_object100738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e482)", 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:57:55.382912+0000", 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 41.348145991999999, 2024-09-13T14:58:36.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.24014121099999999, 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.37095", 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/1312050081", 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 227042 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382912+0000", 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382912+0000", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382910+0000", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382918+0000", 2024-09-13T14:58:36.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6090000000000003e-06 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382935+0000", 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6866000000000001e-05 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382941+0000", 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2230000000000004e-06 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.382985+0000", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.4363000000000002e-05 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.383097+0000", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0001114 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.383208+0000", 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111193 2024-09-13T14:58:36.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.386668+0000", 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0034596620000000001 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.388052+0000", 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0013844650000000001 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.622995+0000", 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.23494335399999999 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.623030+0000", 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4390000000000001e-05 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:57:55.623054+0000", 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3808e-05 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.896 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.897 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:36.897 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:36.897 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:36.951 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:36.972 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:37.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:37.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.220 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:37.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:37.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:37.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:37.446 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:37.467 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:37.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.590 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:37.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:37.692 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:37] "GET /metrics HTTP/1.1" 200 36707 "" "Prometheus/2.51.0" 2024-09-13T14:58:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: 1.1b scrub starts 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: 1.c scrub starts 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: 1.c scrub ok 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: 1.7 scrub starts 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[103001]: 1.7 scrub ok 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: 1.1b scrub starts 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: 1.c scrub starts 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: 1.c scrub ok 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: 1.7 scrub starts 2024-09-13T14:58:37.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:37 smithi017 ceph-mon[107556]: 1.7 scrub ok 2024-09-13T14:58:37.713 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: 1.1b scrub starts 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: 1.c scrub starts 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: 1.c scrub ok 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: 1.7 scrub starts 2024-09-13T14:58:37.746 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:37 smithi189 ceph-mon[90992]: 1.7 scrub ok 2024-09-13T14:58:37.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:58:37.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:37.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:37.939 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:37.961 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:38.060 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:38.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:38.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:38.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:38.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:38.310 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.332 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:38.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.465 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: 1.b scrub starts 2024-09-13T14:58:38.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: 1.b scrub ok 2024-09-13T14:58:38.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:58:38.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: pgmap v1625: 103 pgs: 4 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:38.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: 1.1e scrub starts 2024-09-13T14:58:38.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:38 smithi189 ceph-mon[90992]: 1.1e scrub ok 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: 1.b scrub starts 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: 1.b scrub ok 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: pgmap v1625: 103 pgs: 4 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: 1.1e scrub starts 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[103001]: 1.1e scrub ok 2024-09-13T14:58:38.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: 1.b scrub starts 2024-09-13T14:58:38.524 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: 1.b scrub ok 2024-09-13T14:58:38.524 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:58:38.524 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: pgmap v1625: 103 pgs: 4 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:38.524 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: 1.1e scrub starts 2024-09-13T14:58:38.524 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:38 smithi017 ceph-mon[107556]: 1.1e scrub ok 2024-09-13T14:58:38.558 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:38.681 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:38.702 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:38.804 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.826 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:38.928 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:38.949 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:39.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:39.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:39.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:39.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:39.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:39.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:39.301 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:39.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:39.426 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.448 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.1d scrub starts 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.1d scrub ok 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.9 scrub starts 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.9 scrub ok 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.19 scrub starts 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: 1.19 scrub ok 2024-09-13T14:58:39.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:39 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2207217645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:39.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:39.571 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:39.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:39.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:39.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.1d scrub starts 2024-09-13T14:58:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.1d scrub ok 2024-09-13T14:58:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.9 scrub starts 2024-09-13T14:58:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.9 scrub ok 2024-09-13T14:58:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.19 scrub starts 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: 1.19 scrub ok 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2207217645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.1d scrub starts 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.1d scrub ok 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.9 scrub starts 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.9 scrub ok 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.19 scrub starts 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: 1.19 scrub ok 2024-09-13T14:58:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:39 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2207217645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:39.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:39.807 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:39.808 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:39.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:39.922 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.945 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:39.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.046 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.069 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:40.170 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.191 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:58:40.266 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:58:40.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:40.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:40.339 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:58:40.342 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:58:40.342 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:58:40.407 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:58:40.411 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:40 smithi189 ceph-mon[90992]: 1.16 scrub starts 2024-09-13T14:58:40.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:40 smithi189 ceph-mon[90992]: 1.16 scrub ok 2024-09-13T14:58:40.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:40 smithi189 ceph-mon[90992]: pgmap v1626: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 30 KiB/s wr, 8 op/s; 341 B/s, 0 objects/s recovering 2024-09-13T14:58:40.412 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:40.430 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.436 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:58:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.453 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:40.461 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:58:40.507 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:58:40.509 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:58:40.509 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:58:40.519 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:40.554 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:40.556 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:58:40.559 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-13T14:58:40.559 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:58:40.576 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:40.638 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-13T14:58:40.639 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-13T14:58:40.639 DEBUG:teuthology.orchestra.run.smithi017:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-13T14:58:40.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[103001]: 1.16 scrub starts 2024-09-13T14:58:40.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[103001]: 1.16 scrub ok 2024-09-13T14:58:40.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[103001]: pgmap v1626: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 30 KiB/s wr, 8 op/s; 341 B/s, 0 objects/s recovering 2024-09-13T14:58:40.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[107556]: 1.16 scrub starts 2024-09-13T14:58:40.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[107556]: 1.16 scrub ok 2024-09-13T14:58:40.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:40 smithi017 ceph-mon[107556]: pgmap v1626: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 30 KiB/s wr, 8 op/s; 341 B/s, 0 objects/s recovering 2024-09-13T14:58:40.678 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:40.700 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:40.702 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:40.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.824 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.835 INFO:tasks.workunit.client.0.smithi017.stdout:Running main() from gmock_main.cc 2024-09-13T14:58:40.835 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-13T14:58:40.835 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment set-up. 2024-09-13T14:58:40.835 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd 2024-09-13T14:58:40.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.948 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:40.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.050 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:41.072 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:41.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:41.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:41.259 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 6, 7, 1, 5] 2024-09-13T14:58:41.260 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:41.260 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T14:58:41.260 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:58:41.260 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:58:41.260 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T14:58:41.298 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:41.325 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:41.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:41.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:41.552 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:41.574 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:41.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:41.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[103001]: 1.5 scrub starts 2024-09-13T14:58:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[103001]: 1.5 scrub ok 2024-09-13T14:58:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[103001]: 1.6 scrub starts 2024-09-13T14:58:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[103001]: 1.6 scrub ok 2024-09-13T14:58:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[107556]: 1.5 scrub starts 2024-09-13T14:58:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[107556]: 1.5 scrub ok 2024-09-13T14:58:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[107556]: 1.6 scrub starts 2024-09-13T14:58:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:41 smithi017 ceph-mon[107556]: 1.6 scrub ok 2024-09-13T14:58:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:41 smithi189 ceph-mon[90992]: 1.5 scrub starts 2024-09-13T14:58:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:41 smithi189 ceph-mon[90992]: 1.5 scrub ok 2024-09-13T14:58:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:41 smithi189 ceph-mon[90992]: 1.6 scrub starts 2024-09-13T14:58:41.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:41 smithi189 ceph-mon[90992]: 1.6 scrub ok 2024-09-13T14:58:41.801 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:41.823 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:41.925 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.947 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:42.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:42.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:42.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:42.174 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.197 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:42.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:42.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:42.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:42.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:42.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: 1.a scrub starts 2024-09-13T14:58:42.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: 1.a scrub ok 2024-09-13T14:58:42.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:58:42.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:42.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: pgmap v1628: 135 pgs: 32 unknown, 103 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 31 KiB/s wr, 8 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:42.409 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:42.413 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-13T14:58:42.414 INFO:teuthology.orchestra.run.smithi189.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:42.414 INFO:teuthology.orchestra.run.smithi189.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:42.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:42.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:42.448 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_all_features (34 ms) 2024-09-13T14:58:42.448 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.copyup 2024-09-13T14:58:42.549 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.553 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:42.570 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:42.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:42.634 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.copyup (188 ms) 2024-09-13T14:58:42.683 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-13T14:58:42.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:42.684 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparse_copyup (15 ms) 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-13T14:58:42.685 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: 1.a scrub starts 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: 1.a scrub ok 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: pgmap v1628: 135 pgs: 32 unknown, 103 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 31 KiB/s wr, 8 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: 1.a scrub starts 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: 1.a scrub ok 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: pgmap v1628: 135 pgs: 32 unknown, 103 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 31 KiB/s wr, 8 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:42.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:42.695 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.directory_methods (27 ms) 2024-09-13T14:58:42.695 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.create 2024-09-13T14:58:42.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:42.726 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.create (32 ms) 2024-09-13T14:58:42.726 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_features 2024-09-13T14:58:42.728 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2024-09-13T14:58:42.728 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-13T14:58:42.730 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-13T14:58:42.730 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-13T14:58:42.731 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2024-09-13T14:58:42.731 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-13T14:58:42.732 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-13T14:58:42.733 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-13T14:58:42.734 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-13T14:58:42.734 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-13T14:58:42.737 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-09-13T14:58:42.737 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_size 2024-09-13T14:58:42.742 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-13T14:58:42.742 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_size 2024-09-13T14:58:42.746 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-09-13T14:58:42.746 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-13T14:58:42.763 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.protection_status (16 ms) 2024-09-13T14:58:42.763 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-13T14:58:42.770 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-13T14:58:42.770 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-13T14:58:42.813 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.835 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:42.835 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v1 (65 ms) 2024-09-13T14:58:42.835 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-13T14:58:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:42.865 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2024-09-13T14:58:42.865 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-13T14:58:42.887 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2024-09-13T14:58:42.887 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-13T14:58:42.891 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapid_race (3 ms) 2024-09-13T14:58:42.891 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-13T14:58:42.900 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2024-09-13T14:58:42.901 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-13T14:58:42.902 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2024-09-13T14:58:42.902 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-13T14:58:42.906 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-13T14:58:42.907 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-13T14:58:42.911 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2024-09-13T14:58:42.911 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-13T14:58:42.911 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-13T14:58:42.911 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-13T14:58:42.916 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2024-09-13T14:58:42.917 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-13T14:58:42.923 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_remove (6 ms) 2024-09-13T14:58:42.923 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.flags 2024-09-13T14:58:42.927 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-13T14:58:42.927 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.metadata 2024-09-13T14:58:42.937 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.955 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.metadata (28 ms) 2024-09-13T14:58:42.955 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_features 2024-09-13T14:58:42.960 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:42.962 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-13T14:58:42.962 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror 2024-09-13T14:58:42.996 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror (34 ms) 2024-09-13T14:58:42.996 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-13T14:58:43.011 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2024-09-13T14:58:43.011 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-13T14:58:43.062 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.068 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:58:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.083 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:43.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:43.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:43.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:43.405 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:58:43.443 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:43.465 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:43.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: 1.8 scrub starts 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: 1.8 scrub ok 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/257953235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:43 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.694 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: 1.8 scrub starts 2024-09-13T14:58:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: 1.8 scrub ok 2024-09-13T14:58:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/257953235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: 1.8 scrub starts 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: 1.8 scrub ok 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1211448880' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-404916-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/257953235' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:43 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:43.716 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:43.808 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:43.818 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.841 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:43.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:44.094 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:44.196 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.219 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:44.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:44.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:44.579 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.600 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: 1.e scrub starts 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: 1.e scrub ok 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:58:44.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[107556]: pgmap v1631: 135 pgs: 16 unknown, 119 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 722 KiB/s wr, 28 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: 1.e scrub starts 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: 1.e scrub ok 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:58:44.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:44 smithi017 ceph-mon[103001]: pgmap v1631: 135 pgs: 16 unknown, 119 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 722 KiB/s wr, 28 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:44.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.730 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-13T14:58:44.731 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:44.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: 1.e scrub starts 2024-09-13T14:58:44.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: 1.e scrub ok 2024-09-13T14:58:44.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:44.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:44.731 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:58:44.732 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:44 smithi189 ceph-mon[90992]: pgmap v1631: 135 pgs: 16 unknown, 119 active+clean; 4.8 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 722 KiB/s wr, 28 op/s; 349 B/s, 0 objects/s recovering 2024-09-13T14:58:44.753 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-13T14:58:44.753 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:44.774 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-13T14:58:44.775 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:44.797 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-13T14:58:44.797 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:44.818 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-13T14:58:44.819 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:44.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-13T14:58:44.839 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:44.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:44.862 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-13T14:58:44.862 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:44.883 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-13T14:58:44.883 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:44.961 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.983 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:45.085 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.106 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:45.207 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.228 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:45.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:45.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:45.458 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.479 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:45.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:45.522 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_status (2511 ms) 2024-09-13T14:58:45.523 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-13T14:58:45.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.607 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_map (84 ms) 2024-09-13T14:58:45.607 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-13T14:58:45.614 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2024-09-13T14:58:45.614 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-13T14:58:45.649 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_snapshot (35 ms) 2024-09-13T14:58:45.649 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-13T14:58:45.651 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2024-09-13T14:58:45.651 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-13T14:58:45.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:45 smithi189 ceph-mon[90992]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:58:45.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:45 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:58:45.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:45 smithi189 ceph-mon[90992]: 1.d scrub starts 2024-09-13T14:58:45.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:45 smithi189 ceph-mon[90992]: 1.d scrub ok 2024-09-13T14:58:45.652 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1640412453' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:45.653 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-13T14:58:45.653 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-13T14:58:45.656 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.dir_add_already_existing (2 ms) 2024-09-13T14:58:45.656 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-13T14:58:45.661 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-09-13T14:58:45.661 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-13T14:58:45.664 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-13T14:58:45.664 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-13T14:58:45.667 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-13T14:58:45.667 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-13T14:58:45.669 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-13T14:58:45.669 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-13T14:58:45.673 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-13T14:58:45.673 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-13T14:58:45.675 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-13T14:58:45.675 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-13T14:58:45.679 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-13T14:58:45.679 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-13T14:58:45.681 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-13T14:58:45.681 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-13T14:58:45.684 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-13T14:58:45.684 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-13T14:58:45.686 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-13T14:58:45.686 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-13T14:58:45.688 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-13T14:58:45.688 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-13T14:58:45.690 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-13T14:58:45.690 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-13T14:58:45.693 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-13T14:58:45.693 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-13T14:58:45.696 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-13T14:58:45.696 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1627-gbbdadfa3/rpm/el9/BUILD/ceph-17.2.7-1627-gbbdadfa3/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout:Expected equality of these values: 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout: 1U 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 1 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout: keys.size() 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 3 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-13T14:58:45.698 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-13T14:58:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[107556]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:58:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:58:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[107556]: 1.d scrub starts 2024-09-13T14:58:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[107556]: 1.d scrub ok 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1640412453' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[103001]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[103001]: 1.d scrub starts 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[103001]: 1.d scrub ok 2024-09-13T14:58:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1640412453' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:45.701 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-13T14:58:45.702 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-13T14:58:45.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.716 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2024-09-13T14:58:45.716 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-13T14:58:45.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:45.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:45.909 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (192 ms) 2024-09-13T14:58:45.909 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-13T14:58:45.912 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-09-13T14:58:45.912 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-13T14:58:45.914 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-13T14:58:45.914 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-13T14:58:45.921 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-13T14:58:45.921 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.op_features 2024-09-13T14:58:45.928 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2024-09-13T14:58:45.928 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-13T14:58:45.949 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2024-09-13T14:58:45.950 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-13T14:58:45.957 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-13T14:58:45.958 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-13T14:58:45.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:45.968 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-09-13T14:58:45.968 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-13T14:58:45.975 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-13T14:58:45.975 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration 2024-09-13T14:58:45.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:45.986 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration (10 ms) 2024-09-13T14:58:45.986 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-13T14:58:45.993 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2024-09-13T14:58:45.993 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-13T14:58:46.084 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:46.084 INFO:teuthology.orchestra.run.smithi189.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:58:46.084 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:46.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:46.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:46.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:46.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.336 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:46.360 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:46.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:46.586 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:46.608 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[103001]: 1.18 scrub starts 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[103001]: 1.18 scrub ok 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[103001]: 1.1a scrub starts 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[103001]: 1.1a scrub ok 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[103001]: pgmap v1632: 135 pgs: 5 unknown, 130 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 693 KiB/s wr, 93 op/s 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[107556]: 1.18 scrub starts 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[107556]: 1.18 scrub ok 2024-09-13T14:58:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[107556]: 1.1a scrub starts 2024-09-13T14:58:46.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[107556]: 1.1a scrub ok 2024-09-13T14:58:46.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:46 smithi017 ceph-mon[107556]: pgmap v1632: 135 pgs: 5 unknown, 130 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 693 KiB/s wr, 93 op/s 2024-09-13T14:58:46.710 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:46.732 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:46 smithi189 ceph-mon[90992]: 1.18 scrub starts 2024-09-13T14:58:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:46 smithi189 ceph-mon[90992]: 1.18 scrub ok 2024-09-13T14:58:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:46 smithi189 ceph-mon[90992]: 1.1a scrub starts 2024-09-13T14:58:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:46 smithi189 ceph-mon[90992]: 1.1a scrub ok 2024-09-13T14:58:46.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:46 smithi189 ceph-mon[90992]: pgmap v1632: 135 pgs: 5 unknown, 130 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 1.4 MiB/s rd, 693 KiB/s wr, 93 op/s 2024-09-13T14:58:46.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:46.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:46.958 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:46.979 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.080 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.101 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155671 9.13 9:cb1bd516:::benchmark_data_smithi017_339564_object36198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.72027897800001, 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.922842885, 2024-09-13T14:58:47.149 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155671 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.150 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836828+0000", 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836826+0000", 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.151 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836834+0000", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9310000000000008e-06 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836852+0000", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8045000000000001e-05 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836858+0000", 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1260000000000003e-06 2024-09-13T14:58:47.152 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836923+0000", 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5086999999999995e-05 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.836934+0000", 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0295999999999999e-05 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.153 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.749616+0000", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91268234299999995 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.749707+0000", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0661000000000005e-05 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.750137+0000", 2024-09-13T14:58:47.154 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00043054600000000002 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753267+0000", 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031302109999999999 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759531+0000", 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0062633430000000002 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.155 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759578+0000", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7525999999999999e-05 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759626+0000", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7190000000000001e-05 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759671+0000", 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5284999999999998e-05 2024-09-13T14:58:47.156 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155725 9.13 9:cf82d474:::benchmark_data_smithi017_339564_object36252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.67132570400003, 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87416029900000003, 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.157 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155725 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885781+0000", 2024-09-13T14:58:47.158 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885780+0000", 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885786+0000", 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0739999999999998e-06 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.159 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885796+0000", 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0159999999999999e-05 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885803+0000", 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7410000000000003e-06 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.160 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885880+0000", 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7766e-05 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:19.885891+0000", 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0491e-05 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753309+0000", 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86741857200000005 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.161 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753429+0000", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000119943 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.753775+0000", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034539599999999998 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759713+0000", 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059384269999999996 2024-09-13T14:58:47.162 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759810+0000", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.7238000000000007e-05 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759854+0000", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3268e-05 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759900+0000", 2024-09-13T14:58:47.163 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6601999999999999e-05 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759942+0000", 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1205999999999998e-05 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.164 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155861 9.13 9:cb69ae69:::benchmark_data_smithi017_339564_object36388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.545857377, 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75462436099999997, 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155861 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.165 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011250+0000", 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.166 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011248+0000", 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011255+0000", 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6350000000000006e-06 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.167 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011269+0000", 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3552e-05 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011275+0000", 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5579999999999997e-06 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011328+0000", 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2098999999999997e-05 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.168 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.011339+0000", 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0995000000000001e-05 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756394+0000", 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74505518699999995 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.169 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756484+0000", 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.0662999999999996e-05 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.756876+0000", 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00039175800000000003 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.759991+0000", 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031153069999999999 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.170 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.762998+0000", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003006646 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765772+0000", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0027739549999999998 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765832+0000", 2024-09-13T14:58:47.171 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9579e-05 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765874+0000", 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2277999999999997e-05 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.172 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155867 9.13 9:cc843ee5:::benchmark_data_smithi017_339564_object36394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.54318774699999, 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76626357199999995, 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155867 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.173 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013919+0000", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013918+0000", 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.174 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013923+0000", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1409999999999998e-06 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013931+0000", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5890000000000003e-06 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013937+0000", 2024-09-13T14:58:47.175 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6420000000000004e-06 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013987+0000", 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0076e-05 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.013998+0000", 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1126e-05 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.176 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760009+0000", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74601124900000004 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760148+0000", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013879100000000001 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.760558+0000", 2024-09-13T14:58:47.177 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00040973499999999999 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772815+0000", 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012256800999999999 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772896+0000", 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.1201999999999999e-05 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.178 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780071+0000", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071757339999999996 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780126+0000", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5019999999999998e-05 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780183+0000", 2024-09-13T14:58:47.179 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6257e-05 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155874 9.13 9:cacd9d19:::benchmark_data_smithi017_339564_object36401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.53502798, 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75826068499999999, 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.180 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155874 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022079+0000", 2024-09-13T14:58:47.181 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022078+0000", 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022082+0000", 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8249999999999998e-06 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.182 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022091+0000", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7590000000000003e-06 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022095+0000", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5719999999999996e-06 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022151+0000", 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6004000000000001e-05 2024-09-13T14:58:47.183 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.022163+0000", 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1184e-05 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763022+0000", 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74085988400000002 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.184 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763131+0000", 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010882699999999999 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.763494+0000", 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00036248500000000001 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772846+0000", 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0093522559999999998 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.185 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772933+0000", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7355999999999995e-05 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780239+0000", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0073059930000000002 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780269+0000", 2024-09-13T14:58:47.186 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0113000000000001e-05 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780340+0000", 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0186999999999997e-05 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155911 9.13 9:c90ad918:::benchmark_data_smithi017_339564_object36438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.50200844699998, 2024-09-13T14:58:47.187 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.725374674, 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155911 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.188 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055098+0000", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055097+0000", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055113+0000", 2024-09-13T14:58:47.189 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6109000000000001e-05 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055131+0000", 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7592e-05 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055136+0000", 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8289999999999996e-06 2024-09-13T14:58:47.190 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055304+0000", 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000167226 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.055313+0000", 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.8989999999999996e-06 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.191 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.765911+0000", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71059752200000004 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.766019+0000", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00010836900000000001 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.766446+0000", 2024-09-13T14:58:47.192 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042646300000000003 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772886+0000", 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064403350000000002 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772937+0000", 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1322000000000001e-05 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.193 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780396+0000", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0074582590000000001 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780439+0000", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3310999999999999e-05 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780473+0000", 2024-09-13T14:58:47.194 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4119999999999999e-05 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155920 9.13 9:c9d1c594:::benchmark_data_smithi017_339564_object36447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.49535714299998, 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71884768499999996, 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.195 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155920 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.196 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061750+0000", 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061749+0000", 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.197 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061755+0000", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9710000000000001e-06 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061765+0000", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.9739999999999997e-06 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061771+0000", 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0390000000000001e-06 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.198 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061846+0000", 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5196000000000003e-05 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.061856+0000", 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0091999999999999e-05 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.199 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.769458+0000", 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70760195599999998 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.769570+0000", 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00011196000000000001 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.200 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.770032+0000", 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000462285 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772928+0000", 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0028957190000000002 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.201 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772941+0000", 2024-09-13T14:58:47.202 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3030999999999999e-05 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780534+0000", 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075933520000000003 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.205 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780566+0000", 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1912000000000002e-05 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780597+0000", 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1151e-05 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.206 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155925 9.13 9:cbc0753f:::benchmark_data_smithi017_339564_object36452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.490814462, 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.72457860200000002, 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155925 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.207 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066292+0000", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066291+0000", 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:58:47.208 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066298+0000", 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8480000000000003e-06 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066307+0000", 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2660000000000007e-06 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.209 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066312+0000", 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0150000000000003e-06 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066367+0000", 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.5371000000000002e-05 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.066381+0000", 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4102e-05 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.210 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.772961+0000", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70658006699999998 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.773077+0000", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000115244 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.773527+0000", 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000449901 2024-09-13T14:58:47.211 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780508+0000", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0069818889999999998 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780650+0000", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014148399999999999 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790742+0000", 2024-09-13T14:58:47.212 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010092131000000001 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790832+0000", 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9962000000000004e-05 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790871+0000", 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9002999999999999e-05 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.213 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155944 9.13 9:cb290f3d:::benchmark_data_smithi017_339564_object36471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.47261294899999, 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70647894899999997, 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.214 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155944 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084494+0000", 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.215 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084493+0000", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084497+0000", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.455e-06 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084505+0000", 2024-09-13T14:58:47.216 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.7370000000000006e-06 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084509+0000", 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.228e-06 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084552+0000", 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3189999999999998e-05 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.217 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.084561+0000", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9779999999999994e-06 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776212+0000", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69165086600000003 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776296+0000", 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3761e-05 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.218 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.776736+0000", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00044013400000000001 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780625+0000", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0038895140000000002 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780677+0000", 2024-09-13T14:58:47.219 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1394000000000003e-05 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790879+0000", 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010202573 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790938+0000", 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9088000000000001e-05 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.220 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.790973+0000", 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4690999999999997e-05 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:155948 9.13 9:c80b3a9a:::benchmark_data_smithi017_339564_object36475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 386.46875726399998, 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.71039708300000004, 2024-09-13T14:58:47.221 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 155948 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.222 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088350+0000", 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088349+0000", 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.223 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088352+0000", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3859999999999999e-06 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088364+0000", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1943000000000001e-05 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088369+0000", 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7129999999999996e-06 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.224 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088416+0000", 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6897000000000002e-05 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.088423+0000", 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8619999999999997e-06 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.225 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780695+0000", 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.69227191700000001 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.780783+0000", 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7839999999999999e-05 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.781237+0000", 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00045437599999999999 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.226 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.791023+0000", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0097855939999999999 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798583+0000", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0075606090000000003 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798663+0000", 2024-09-13T14:58:47.227 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9684999999999993e-05 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798712+0000", 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8634999999999998e-05 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:20.798747+0000", 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5209000000000001e-05 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.228 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:234956 9.15 9:a9c4d06b:::benchmark_data_smithi017_339564_object115483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.86292134799999, 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.94212623900000003, 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 234956 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.229 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694186+0000", 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.230 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694184+0000", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694190+0000", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9440000000000002e-06 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694204+0000", 2024-09-13T14:58:47.231 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3579e-05 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694209+0000", 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4559999999999999e-06 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694263+0000", 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3343000000000001e-05 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.232 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.694272+0000", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4550000000000001e-06 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611453+0000", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91718084799999999 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611537+0000", 2024-09-13T14:58:47.233 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.3949000000000005e-05 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.611945+0000", 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00040808199999999999 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616712+0000", 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047666710000000001 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.234 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636236+0000", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019524308000000001 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636262+0000", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5935000000000002e-05 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636290+0000", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8626999999999999e-05 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636312+0000", 2024-09-13T14:58:47.235 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1372000000000001e-05 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235016 9.15 9:a9ffcdb4:::benchmark_data_smithi017_339564_object115543:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.81080621199999, 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.890119416, 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.236 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235016 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.237 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746301+0000", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746300+0000", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746306+0000", 2024-09-13T14:58:47.238 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8229999999999997e-06 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746318+0000", 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2009e-05 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746325+0000", 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4420000000000003e-06 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.239 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746861+0000", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053583600000000004 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.746869+0000", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2269999999999993e-06 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.613740+0000", 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86687114600000004 2024-09-13T14:58:47.240 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.613811+0000", 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.148e-05 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.614076+0000", 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00026431000000000002 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.241 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618157+0000", 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.004081459 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636349+0000", 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018191807000000001 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636383+0000", 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3850000000000003e-05 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.242 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636404+0000", 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1271999999999999e-05 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636420+0000", 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5965e-05 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.243 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235041 9.15 9:ae766b01:::benchmark_data_smithi017_339564_object115568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.78771608099999, 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86726312000000005, 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235041 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.244 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769391+0000", 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.245 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769390+0000", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769396+0000", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7219999999999996e-06 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769406+0000", 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0538e-05 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.246 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.769412+0000", 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7130000000000002e-06 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771644+0000", 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002232506 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.247 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771739+0000", 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.4989999999999997e-05 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615272+0000", 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84353275999999999 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615337+0000", 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4394999999999996e-05 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.248 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.615557+0000", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022054600000000001 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622593+0000", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070358640000000002 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636445+0000", 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013852491999999999 2024-09-13T14:58:47.249 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636557+0000", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000111919 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636621+0000", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4115999999999994e-05 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636654+0000", 2024-09-13T14:58:47.250 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2498e-05 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235042 9.15 9:a8280366:::benchmark_data_smithi017_339564_object115569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.787069729, 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.866749561, 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.251 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235042 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770037+0000", 2024-09-13T14:58:47.252 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770036+0000", 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770041+0000", 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1560000000000003e-06 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.253 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770057+0000", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5051999999999999e-05 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.770063+0000", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2519999999999996e-06 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771761+0000", 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001698101 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.254 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.771767+0000", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.7259999999999999e-06 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616749+0000", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.84498218899999999 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.616807+0000", 2024-09-13T14:58:47.255 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8288999999999997e-05 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.617048+0000", 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00024090799999999999 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625508+0000", 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084598209999999993 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.256 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636701+0000", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011193053999999999 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636734+0000", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3061000000000001e-05 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636757+0000", 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3300000000000001e-05 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.257 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636787+0000", 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9543000000000002e-05 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235059 9.15 9:ab89eb0d:::benchmark_data_smithi017_339564_object115586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.77065180699998, 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.85055655600000002, 2024-09-13T14:58:47.258 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235059 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.259 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786455+0000", 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786454+0000", 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786459+0000", 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1329999999999998e-06 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786469+0000", 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2299999999999997e-06 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786474+0000", 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8620000000000002e-06 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.261 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786521+0000", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7814e-05 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.786530+0000", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.8040000000000007e-06 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618177+0000", 2024-09-13T14:58:47.262 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83164686700000001 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618251+0000", 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4198999999999999e-05 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.618489+0000", 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000237875 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.263 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625510+0000", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070213799999999998 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636836+0000", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011325426 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636936+0000", 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000100246 2024-09-13T14:58:47.264 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636979+0000", 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3198999999999998e-05 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637012+0000", 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.2322e-05 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.265 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235072 9.15 9:ad20fbe1:::benchmark_data_smithi017_339564_object115599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.75904811499998, 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83910899999999999, 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235072 2024-09-13T14:58:47.266 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798059+0000", 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.267 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798058+0000", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798064+0000", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798075+0000", 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1813e-05 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.268 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798081+0000", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3900000000000001e-06 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798135+0000", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4280999999999997e-05 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.798144+0000", 2024-09-13T14:58:47.269 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6780000000000006e-06 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619664+0000", 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.82152004199999995 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619724+0000", 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9849999999999998e-05 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.270 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.619938+0000", 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021460899999999999 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625513+0000", 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0055745090000000001 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.636890+0000", 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011376828 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637105+0000", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00021505100000000001 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637133+0000", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8121999999999999e-05 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637168+0000", 2024-09-13T14:58:47.272 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4981000000000003e-05 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235104 9.15 9:ad7f5d5c:::benchmark_data_smithi017_339564_object115631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.72668239799998, 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80683303200000001, 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.273 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235104 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.274 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830425+0000", 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830424+0000", 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830428+0000", 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8120000000000001e-06 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.275 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830436+0000", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.0900000000000005e-06 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830440+0000", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6729999999999998e-06 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830481+0000", 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1090000000000001e-05 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.276 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.830487+0000", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6869999999999997e-06 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621047+0000", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.790559503 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621112+0000", 2024-09-13T14:58:47.277 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5216999999999996e-05 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.621423+0000", 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00031051599999999999 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.628538+0000", 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071157879999999996 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.278 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637070+0000", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0085320399999999994 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637199+0000", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012852299999999999 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637224+0000", 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5612000000000001e-05 2024-09-13T14:58:47.279 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637258+0000", 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.3073e-05 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235132 9.15 9:ae3e20fb:::benchmark_data_smithi017_339564_object115659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.69603620999999, 2024-09-13T14:58:47.280 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.77630533999999995, 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235132 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.281 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861071+0000", 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861069+0000", 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.282 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861076+0000", 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1179999999999997e-06 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861086+0000", 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.04e-05 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.861091+0000", 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2650000000000004e-06 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.283 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.862798+0000", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001706611 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.862806+0000", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4090000000000003e-06 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622623+0000", 2024-09-13T14:58:47.284 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75981690300000004 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622690+0000", 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6616000000000001e-05 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.622966+0000", 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00027669599999999999 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.285 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.628541+0000", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.005574603 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637286+0000", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0087450630000000008 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637309+0000", 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3176e-05 2024-09-13T14:58:47.286 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637332+0000", 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3040999999999998e-05 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637376+0000", 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3698000000000001e-05 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.287 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235144 9.15 9:af6e4b6c:::benchmark_data_smithi017_339564_object115671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.68161774399999, 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76209903800000001, 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235144 2024-09-13T14:58:47.288 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875489+0000", 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.289 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875488+0000", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875494+0000", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2700000000000004e-06 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875504+0000", 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0808999999999999e-05 2024-09-13T14:58:47.290 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875509+0000", 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3660000000000002e-06 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875549+0000", 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0790000000000001e-05 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.291 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.875555+0000", 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8629999999999996e-06 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624121+0000", 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.748565641 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624186+0000", 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5436000000000002e-05 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.292 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.624422+0000", 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000235236 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.631482+0000", 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0070606430000000001 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.293 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637431+0000", 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059484849999999999 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637507+0000", 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6427999999999994e-05 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637549+0000", 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1567999999999999e-05 2024-09-13T14:58:47.294 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637588+0000", 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9459000000000003e-05 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:235167 9.15 9:a8ab814f:::benchmark_data_smithi017_339564_object115694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 292.65465729200002, 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.73526514799999998, 2024-09-13T14:58:47.295 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 235167 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.296 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902450+0000", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902448+0000", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902455+0000", 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4629999999999998e-06 2024-09-13T14:58:47.297 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902471+0000", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5750999999999999e-05 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902477+0000", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.857e-06 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902526+0000", 2024-09-13T14:58:47.298 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8964000000000001e-05 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:53.902534+0000", 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.456e-06 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625559+0000", 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.72302467800000003 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.299 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625624+0000", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5321999999999996e-05 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.625900+0000", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00027566800000000002 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.631502+0000", 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0056024730000000002 2024-09-13T14:58:47.300 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637469+0000", 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0059668270000000001 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637640+0000", 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00017078699999999999 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:58:47.301 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637681+0000", 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1028000000000001e-05 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:54.637715+0000", 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4029999999999998e-05 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:58:47.302 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:58:47.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:47.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:47.423 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1429 ms) 2024-09-13T14:58:47.423 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-13T14:58:47.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:47.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:47.464 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparsify (42 ms) 2024-09-13T14:58:47.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:47.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:47.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:47] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[103001]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[103001]: 1.15 scrub starts 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[103001]: 1.15 scrub ok 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[107556]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[107556]: 1.15 scrub starts 2024-09-13T14:58:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:47 smithi017 ceph-mon[107556]: 1.15 scrub ok 2024-09-13T14:58:47.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:47 smithi189 ceph-mon[90992]: osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:58:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:47 smithi189 ceph-mon[90992]: 1.15 scrub starts 2024-09-13T14:58:47.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:47 smithi189 ceph-mon[90992]: 1.15 scrub ok 2024-09-13T14:58:47.806 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.828 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:58:47.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:47.952 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:47.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:58:48.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:48.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:48.141 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:48.180 DEBUG:teuthology.orchestra.run.smithi189:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.202 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: docker: command not found 2024-09-13T14:58:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi189 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:48.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.440 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd (5052 ms total) 2024-09-13T14:58:48.440 INFO:tasks.workunit.client.0.smithi017.stdout: 2024-09-13T14:58:48.440 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment tear-down 2024-09-13T14:58:48.440 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] 70 tests from 1 test suite ran. (7605 ms total) 2024-09-13T14:58:48.441 INFO:tasks.workunit.client.0.smithi017.stdout:[ PASSED ] 69 tests. 2024-09-13T14:58:48.441 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] 1 test, listed below: 2024-09-13T14:58:48.441 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-13T14:58:48.441 INFO:tasks.workunit.client.0.smithi017.stdout: 2024-09-13T14:58:48.441 INFO:tasks.workunit.client.0.smithi017.stdout: 1 FAILED TEST 2024-09-13T14:58:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.443 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-13T14:58:48.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:58:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:48.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:48.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:48.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-13T14:58:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:48.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:48 smithi017 ceph-mon[103001]: osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:58:48.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:48 smithi017 ceph-mon[103001]: pgmap v1635: 135 pgs: 135 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 826 KiB/s wr, 171 op/s 2024-09-13T14:58:48.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:48 smithi017 ceph-mon[107556]: osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:58:48.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:48 smithi017 ceph-mon[107556]: pgmap v1635: 135 pgs: 135 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 826 KiB/s wr, 171 op/s 2024-09-13T14:58:48.682 INFO:tasks.thrashosds:joining thrashosds 2024-09-13T14:58:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:48 smithi189 ceph-mon[90992]: osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:58:48.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:48 smithi189 ceph-mon[90992]: pgmap v1635: 135 pgs: 135 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.7 MiB/s rd, 826 KiB/s wr, 171 op/s 2024-09-13T14:58:48.910 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:49 smithi017 ceph-mon[103001]: osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:58:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:49 smithi017 ceph-mon[107556]: osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:58:49.756 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:49 smithi189 ceph-mon[90992]: osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:58:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:50 smithi189 ceph-mon[90992]: 1.1b scrub starts 2024-09-13T14:58:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:50 smithi189 ceph-mon[90992]: 1.1b scrub ok 2024-09-13T14:58:50.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:50 smithi189 ceph-mon[90992]: pgmap v1637: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:58:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:58:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[103001]: 1.1b scrub starts 2024-09-13T14:58:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[103001]: 1.1b scrub ok 2024-09-13T14:58:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[103001]: pgmap v1637: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[107556]: 1.1b scrub starts 2024-09-13T14:58:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[107556]: 1.1b scrub ok 2024-09-13T14:58:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:50 smithi017 ceph-mon[107556]: pgmap v1637: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:51.967 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:58:51.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-09-13T14:58:52.347 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:58:52.347 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:58:52.347 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-13T14:58:52.347 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:58:52.555 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-13T14:58:52.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[103001]: 1.1c scrub starts 2024-09-13T14:58:52.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[103001]: 1.1c scrub ok 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[103001]: pgmap v1638: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[107556]: 1.1c scrub starts 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[107556]: 1.1c scrub ok 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[107556]: pgmap v1638: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:52.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:52 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:52 smithi189 ceph-mon[90992]: 1.1c scrub starts 2024-09-13T14:58:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:52 smithi189 ceph-mon[90992]: 1.1c scrub ok 2024-09-13T14:58:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:52 smithi189 ceph-mon[90992]: pgmap v1638: 103 pgs: 103 active+clean; 4.7 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:52 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:53.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:58:53.827 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:53.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[103001]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:53.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:53.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[103001]: osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:58:53.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:53.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[107556]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:53.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:53.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[107556]: osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:58:53.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:53 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:53 smithi189 ceph-mon[90992]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2160089083' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:53 smithi189 ceph-mon[90992]: osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:58:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:53 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:58:54.536 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:54 smithi017 ceph-mon[103001]: pgmap v1640: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:54 smithi017 ceph-mon[107556]: pgmap v1640: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:54 smithi189 ceph-mon[90992]: pgmap v1640: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:55.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:58:56.658 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 31 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[103001]: pgmap v1641: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1097541652' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[107556]: pgmap v1641: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1097541652' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:56 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:56.998 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:57.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:56 smithi189 ceph-mon[90992]: pgmap v1641: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:56 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1097541652' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:56 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-13T14:58:57.621 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:58:57] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.51.0" 2024-09-13T14:58:57.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-13T14:58:57.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T14:58:57.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[103001]: osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:58:57.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:57.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T14:58:57.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[107556]: osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:58:57.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-13T14:58:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:57 smithi189 ceph-mon[90992]: osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:58:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:58.543 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:58:58.543 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-13T14:58:58.543 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-13T14:58:58.543 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-13T14:58:58.544 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:58:58.897 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:58 smithi017 ceph-mon[107556]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:58.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:58 smithi017 ceph-mon[103001]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:58 smithi189 ceph-mon[90992]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:58:59.853 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:59.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:58:59.985 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:58:59 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:59.986 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:58:59 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:59:00.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:58:59 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:59:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:00.853 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-13T14:59:01.112 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:00 smithi189 ceph-mon[90992]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:01.112 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/4168286867' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.112 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:00 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[107556]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/4168286867' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[103001]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/4168286867' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:00 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:01.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:59:02.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T14:59:02.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: Cluster is now healthy 2024-09-13T14:59:02.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:02.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:59:02.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: 1.17 scrub starts 2024-09-13T14:59:02.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:01 smithi189 ceph-mon[90992]: 1.17 scrub ok 2024-09-13T14:59:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T14:59:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: Cluster is now healthy 2024-09-13T14:59:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: 1.17 scrub starts 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[103001]: 1.17 scrub ok 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: Cluster is now healthy 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: 1.17 scrub starts 2024-09-13T14:59:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:01 smithi017 ceph-mon[107556]: 1.17 scrub ok 2024-09-13T14:59:02.208 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:03.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:02 smithi017 ceph-mon[107556]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:03.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:02 smithi017 ceph-mon[103001]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:03.177 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:02 smithi189 ceph-mon[90992]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:03.392 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2024-09-13T14:59:03.392 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2024-09-13T14:59:03.392 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2024-09-13T14:59:03.393 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:59:03.393 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:59:03.393 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:59:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/929885532' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:59:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/929885532' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:59:04.126 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/929885532' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:59:04.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[103001]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:04.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:04.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:04.875 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[107556]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:04 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:05.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:04 smithi189 ceph-mon[90992]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:05.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:05.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:04 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:05.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:59:06.686 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-13T14:59:06.686 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 476.394341838, 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32587973199999998, 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.687 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:59:06.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786708+0000", 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786731+0000", 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2778000000000001e-05 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786747+0000", 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6589e-05 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.786753+0000", 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7810000000000002e-06 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792834+0000", 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060810639999999997 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.792996+0000", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016137599999999999 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.793158+0000", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000162034 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.797490+0000", 2024-09-13T14:59:06.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043325070000000002 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.805581+0000", 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080903590000000001 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112577+0000", 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30699643500000001 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112587+0000", 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031e-05 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46712.0:95184 9.2 e382/376 9:49f8b0b3:::benchmark_data_smithi017_339564_object95183:head v 382'6052, mlcod=382'6047)", 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 476.38072438099999, 2024-09-13T14:59:06.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35791878900000001, 2024-09-13T14:59:06.696 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.696 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800325+0000", 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800326+0000", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0500000000000002e-07 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800353+0000", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6665e-05 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800365+0000", 2024-09-13T14:59:06.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.205e-05 2024-09-13T14:59:06.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.800374+0000", 2024-09-13T14:59:06.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2024-09-13T14:59:06.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801552+0000", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011782439999999999 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.801573+0000", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0863999999999999e-05 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.158244+0000", 2024-09-13T14:59:06.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35667120099999999 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95182 9.0 9:0107e64f:::benchmark_data_smithi017_339564_object95181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 476.26234062600003, 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.87911086900000002, 2024-09-13T14:59:06.702 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.703 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95182 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918709+0000", 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918711+0000", 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5579999999999999e-06 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918751+0000", 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9842999999999998e-05 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918774+0000", 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2779999999999999e-05 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918785+0000", 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112614+0000", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19382977600000001 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112623+0000", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0620000000000007e-06 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.796000+0000", 2024-09-13T14:59:06.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68337630999999999 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.797820+0000", 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00182003 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:95183 9.0 9:030aab3b:::benchmark_data_smithi017_339564_object95182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:59:06.708 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 476.26224375700002, 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88068354999999998, 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 95183 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:59:06.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.918806+0000", 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5799999999999999e-07 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919323+0000", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051669300000000004 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919336+0000", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3485999999999999e-05 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:09.919344+0000", 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2024-09-13T14:59:06.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112646+0000", 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19330140800000001 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.112652+0000", 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6499999999999999e-06 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799403+0000", 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.68675042600000002 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:10.799489+0000", 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6649000000000003e-05 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.713 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99743 9.10 9:09e78ae4:::benchmark_data_smithi017_339564_object99742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26937171700001, 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2203931610000001, 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99743 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911678+0000", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911679+0000", 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3689999999999999e-06 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911728+0000", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.922e-05 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911741+0000", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2315e-05 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911746+0000", 2024-09-13T14:59:06.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8080000000000001e-06 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911801+0000", 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4157999999999999e-05 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911808+0000", 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6739999999999997e-06 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102095+0000", 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1902865440000001 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102270+0000", 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017495 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.102476+0000", 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000205787 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109826+0000", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073506869999999998 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109842+0000", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.554e-05 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115907+0000", 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060650799999999996 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129977+0000", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014069813 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.130088+0000", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111799 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132071+0000", 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019824170000000002 2024-09-13T14:59:06.720 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99758 9.10 9:0b28f730:::benchmark_data_smithi017_339564_object99757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26928746499999, 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2224807790000001, 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.721 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99758 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911762+0000", 2024-09-13T14:59:06.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911763+0000", 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2e-07 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911826+0000", 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3499999999999999e-05 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911834+0000", 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9570000000000002e-06 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911838+0000", 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0049999999999998e-06 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911869+0000", 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0925e-05 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911873+0000", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9539999999999998e-06 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105698+0000", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.193825261 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.105855+0000", 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015642 2024-09-13T14:59:06.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.106033+0000", 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017866900000000001 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.109848+0000", 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038150580000000001 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121439+0000", 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01159059 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132121+0000", 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682087999999999 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132209+0000", 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7552000000000005e-05 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132324+0000", 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115606 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134243+0000", 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001918674 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.728 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99769 9.10 9:081d7eac:::benchmark_data_smithi017_339564_object99768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.269200646, 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.223936653, 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99769 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911849+0000", 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8800000000000003e-07 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911912+0000", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2597999999999999e-05 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911920+0000", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4309999999999994e-06 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911924+0000", 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1389999999999997e-06 2024-09-13T14:59:06.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911955+0000", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0182e-05 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911958+0000", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6330000000000002e-06 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.107964+0000", 2024-09-13T14:59:06.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1960057749999999 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108122+0000", 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015787400000000001 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.108353+0000", 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023077800000000001 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115855+0000", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075024380000000002 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121484+0000", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056290380000000003 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.132167+0000", 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010682957 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134290+0000", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021226890000000001 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.134356+0000", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5739000000000002e-05 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135785+0000", 2024-09-13T14:59:06.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014298939999999999 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99779 9.10 9:0b953e14:::benchmark_data_smithi017_339564_object99778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26911432999998, 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.225690441, 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.736 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99779 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911935+0000", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911936+0000", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911991+0000", 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5569999999999998e-05 2024-09-13T14:59:06.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.911999+0000", 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5089999999999996e-06 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912003+0000", 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9319999999999998e-06 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912031+0000", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8237000000000001e-05 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912034+0000", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.444e-06 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110235+0000", 2024-09-13T14:59:06.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.198200776 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110443+0000", 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000207517 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.110617+0000", 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017471500000000001 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115921+0000", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053037340000000001 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135838+0000", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019916776000000001 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135894+0000", 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6515999999999999e-05 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136027+0000", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013228 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.136088+0000", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1495999999999995e-05 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137626+0000", 2024-09-13T14:59:06.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015374570000000001 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99792 9.10 9:0afafc95:::benchmark_data_smithi017_339564_object99791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26903611099999, 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2470098199999999, 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.744 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99792 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912013+0000", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912014+0000", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7599999999999997e-07 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912055+0000", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0843999999999997e-05 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912062+0000", 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3640000000000001e-06 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912066+0000", 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5379999999999998e-06 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912091+0000", 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5678999999999999e-05 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912095+0000", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4240000000000002e-06 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113399+0000", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2013038380000001 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113589+0000", 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019008499999999999 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.113800+0000", 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021092600000000001 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121388+0000", 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007588086 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135939+0000", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014551708 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137677+0000", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001737527 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156695+0000", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019018576999999998 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156755+0000", 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0028000000000002e-05 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159023+0000", 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00226772 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99812 9.10 9:08133f40:::benchmark_data_smithi017_339564_object99811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:59:06.751 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26897614699999, 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2497001430000001, 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99812 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912073+0000", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912074+0000", 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7999999999999996e-07 2024-09-13T14:59:06.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912114+0000", 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9949e-05 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912121+0000", 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0820000000000001e-06 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912124+0000", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4309999999999998e-06 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912148+0000", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3368999999999999e-05 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912151+0000", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.45e-06 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.115948+0000", 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.203796654 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116167+0000", 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021876999999999999 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.116334+0000", 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000167794 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121395+0000", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050604029999999998 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.135984+0000", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014589289 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137719+0000", 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001734684 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159936+0000", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022217263000000001 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.160008+0000", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1608999999999998e-05 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161773+0000", 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001765916 2024-09-13T14:59:06.758 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99825 9.10 9:0c0deca2:::benchmark_data_smithi017_339564_object99824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.268917461, 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2555997640000001, 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99825 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912132+0000", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912171+0000", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8818e-05 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912178+0000", 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9840000000000004e-06 2024-09-13T14:59:06.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912182+0000", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7809999999999999e-06 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912230+0000", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7647e-05 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912240+0000", 2024-09-13T14:59:06.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.005e-05 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118734+0000", 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2064939020000001 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.118892+0000", 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158796 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.119058+0000", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000165336 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129912+0000", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010854278 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145102+0000", 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015189947000000001 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145183+0000", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1160000000000002e-05 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165672+0000", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020488431000000001 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.165735+0000", 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2934000000000004e-05 2024-09-13T14:59:06.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.167732+0000", 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00199723 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99867 9.10 9:08792be1:::benchmark_data_smithi017_339564_object99866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26885964600001, 2024-09-13T14:59:06.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.258038813, 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99867 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912190+0000", 2024-09-13T14:59:06.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7399999999999998e-07 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912235+0000", 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4255999999999998e-05 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912244+0000", 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3999999999999998e-06 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912250+0000", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912279+0000", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8410000000000001e-05 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912283+0000", 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8190000000000002e-06 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121515+0000", 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2092329669999999 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121672+0000", 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015640000000000001 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.121848+0000", 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017612800000000001 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.129919+0000", 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080710069999999998 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145146+0000", 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015227021 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145227+0000", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1304000000000006e-05 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168260+0000", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023032357999999999 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.168415+0000", 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015499000000000001 2024-09-13T14:59:06.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170229+0000", 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018139320000000001 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99888 9.10 9:09cf0b05:::benchmark_data_smithi017_339564_object99887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.268788288, 2024-09-13T14:59:06.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.260453754, 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99888 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912261+0000", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912262+0000", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9900000000000001e-07 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912307+0000", 2024-09-13T14:59:06.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5405999999999999e-05 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912314+0000", 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3220000000000004e-06 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912318+0000", 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7019999999999999e-06 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912344+0000", 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.614e-05 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912348+0000", 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.597e-06 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.123904+0000", 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.211556031 2024-09-13T14:59:06.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124058+0000", 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153933 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.124254+0000", 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019642300000000001 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140317+0000", 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016063137000000002 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152386+0000", 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012068270000000001 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153192+0000", 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080682099999999999 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170291+0000", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017098248 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.170362+0000", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1415999999999995e-05 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172715+0000", 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002352809 2024-09-13T14:59:06.781 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99894 9.10 9:0e8bfb10:::benchmark_data_smithi017_339564_object99893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26872341799998, 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2902429099999999, 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99894 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912326+0000", 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4000000000000002e-07 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912366+0000", 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9403999999999997e-05 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912373+0000", 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0910000000000004e-06 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912377+0000", 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6890000000000002e-06 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912402+0000", 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5584000000000001e-05 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912406+0000", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4419999999999998e-06 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126634+0000", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.214228753 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126788+0000", 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015391299999999999 2024-09-13T14:59:06.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.126969+0000", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018079000000000001 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145055+0000", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018085981000000001 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152432+0000", 2024-09-13T14:59:06.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073765209999999996 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153227+0000", 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079486599999999998 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200627+0000", 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047400281000000002 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.200692+0000", 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5309999999999996e-05 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202569+0000", 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001876845 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99941 9.10 9:0dcd111c:::benchmark_data_smithi017_339564_object99940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:59:06.789 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26866501799998, 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.291876698, 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99941 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:59:06.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912384+0000", 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912385+0000", 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8599999999999998e-07 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912423+0000", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8223000000000003e-05 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912430+0000", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8310000000000003e-06 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912434+0000", 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6679999999999999e-06 2024-09-13T14:59:06.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912459+0000", 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5049999999999999e-05 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912462+0000", 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.439e-06 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137750+0000", 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2252876800000001 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.137916+0000", 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166047 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.138092+0000", 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017649300000000001 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145062+0000", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069691930000000003 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.153258+0000", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081965000000000007 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.156056+0000", 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002798067 2024-09-13T14:59:06.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202632+0000", 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.046576059000000003 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.202696+0000", 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4141999999999997e-05 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204261+0000", 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00156482 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99950 9.10 9:0d57d527:::benchmark_data_smithi017_339564_object99949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26860821299999, 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2935929129999999, 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.797 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99950 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912441+0000", 2024-09-13T14:59:06.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912442+0000", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7e-07 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912480+0000", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8099999999999998e-05 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912487+0000", 2024-09-13T14:59:06.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8990000000000004e-06 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912490+0000", 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912526+0000", 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5649999999999999e-05 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7969999999999998e-06 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.140490+0000", 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2279604200000001 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141138+0000", 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064823500000000004 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.141336+0000", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019801899999999999 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145114+0000", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037779839999999999 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.159122+0000", 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014007400999999999 2024-09-13T14:59:06.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.163411+0000", 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042897459999999997 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204322+0000", 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.040910421000000002 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.204554+0000", 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023251700000000001 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206034+0000", 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014799290000000001 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99954 9.10 9:0ddee9ee:::benchmark_data_smithi017_339564_object99953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.804 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26855226200001, 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2956523129999999, 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99954 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912497+0000", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912498+0000", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.89e-07 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912517+0000", 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9406999999999999e-05 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912525+0000", 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3699999999999995e-06 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912529+0000", 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7469999999999999e-06 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912563+0000", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3584999999999998e-05 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912571+0000", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0949999999999996e-06 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143130+0000", 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.230558984 2024-09-13T14:59:06.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143272+0000", 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000142399 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.143498+0000", 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022541600000000001 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145240+0000", 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017423510000000001 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.161922+0000", 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016681756999999998 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172781+0000", 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010859051 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206100+0000", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033319357000000001 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.206306+0000", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020586899999999999 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208150+0000", 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018434359999999999 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.811 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:99965 9.10 9:081c4e71:::benchmark_data_smithi017_339564_object99964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.268512961, 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2972334430000001, 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.812 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 99965 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912537+0000", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0900000000000002e-07 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912588+0000", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1122000000000003e-05 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912598+0000", 2024-09-13T14:59:06.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0046e-05 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912604+0000", 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9909999999999999e-06 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912650+0000", 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6013000000000001e-05 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912657+0000", 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7429999999999996e-06 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145268+0000", 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2326112730000001 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145425+0000", 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156794 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.145584+0000", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158839 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150428+0000", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048441869999999998 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.172958+0000", 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022530171000000002 2024-09-13T14:59:06.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178611+0000", 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056524990000000001 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208220+0000", 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029609139999999999 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.208377+0000", 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015774100000000001 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209770+0000", 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001392475 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100028 9.10 9:0bded79a:::benchmark_data_smithi017_339564_object100027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.819 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.26843214500002, 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.300064551, 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100028 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912617+0000", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912618+0000", 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:59:06.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912662+0000", 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4375000000000001e-05 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912671+0000", 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5550000000000003e-06 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912676+0000", 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8749999999999999e-06 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912725+0000", 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9576000000000001e-05 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912733+0000", 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4150000000000002e-06 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148624+0000", 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2358912049999999 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148772+0000", 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147727 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.148947+0000", 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175921 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152859+0000", 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039114140000000002 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173020+0000", 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020160900999999998 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178657+0000", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005637268 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.209824+0000", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.031167007 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.210229+0000", 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040503799999999998 2024-09-13T14:59:06.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212682+0000", 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024528060000000001 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46712.0:100040 9.10 9:0f8c354b:::benchmark_data_smithi017_339564_object100039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e386)", 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:59:06.827 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 469.268364416, 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.302457982, 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46712", 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 100040 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-13T14:59:06.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912685+0000", 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3000000000000002e-07 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912719+0000", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3429000000000001e-05 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912724+0000", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9490000000000004e-06 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912727+0000", 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3249999999999999e-06 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912758+0000", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0531000000000003e-05 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:16.912768+0000", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0879e-05 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150460+0000", 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2376911479999999 2024-09-13T14:59:06.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150598+0000", 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013841999999999999 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4,7", 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.150758+0000", 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000159479 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-13T14:59:06.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.152866+0000", 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021080199999999999 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.173065+0000", 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020199238000000001 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.178693+0000", 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056281609999999996 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212776+0000", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034082685000000001 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.212896+0000", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120565 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-13T14:51:18.215143+0000", 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022468229999999998 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.834 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-13T14:59:06.835 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-13T14:59:06.835 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-13T14:59:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:06 smithi017 ceph-mon[103001]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:06 smithi017 ceph-mon[107556]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:07.031 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:07.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:06 smithi189 ceph-mon[90992]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:07.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:07] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.51.0" 2024-09-13T14:59:07.548 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:59:07.808 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:08.909 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:08 smithi189 ceph-mon[90992]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:08 smithi017 ceph-mon[103001]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:08 smithi017 ceph-mon[107556]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:09.509 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:59:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:59:09.509 INFO:teuthology.orchestra.run.smithi189.stdout: "num_ops": 0 2024-09-13T14:59:09.509 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:59:09.880 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-13T14:59:09.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/307842059' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:09.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:09 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:09.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/307842059' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:09.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:09 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:10.109 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/307842059' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:10.109 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:09 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:10.134 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:59:10.381 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:10.502 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:10.881 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[103001]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[103001]: osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1980421475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[107556]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[107556]: osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:59:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1980421475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:10 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-mon[90992]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:11.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:11.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-mon[90992]: osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:59:11.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1980421475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:10 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:11.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-13T14:59:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:11 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:11.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:11 smithi017 ceph-mon[103001]: osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:59:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:11 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:11 smithi017 ceph-mon[107556]: osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:59:12.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:11 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:12.130 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:11 smithi189 ceph-mon[90992]: osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:59:12.159 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:59:12.159 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [], 2024-09-13T14:59:12.159 INFO:teuthology.orchestra.run.smithi189.stdout: "complaint_time": 30, 2024-09-13T14:59:12.159 INFO:teuthology.orchestra.run.smithi189.stdout: "num_blocked_ops": 0 2024-09-13T14:59:12.159 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:59:12.521 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:12.844 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:59:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[103001]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:12.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[107556]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:13.091 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:13.212 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:12 smithi189 ceph-mon[90992]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:13.212 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:13.212 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:14.864 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-09-13T14:59:14.864 INFO:teuthology.orchestra.run.smithi189.stdout: "size": 20, 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 600, 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "ops": [ 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95176 9.7 9:e2f344fa:::benchmark_data_smithi017_339564_object95175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 484.94354997800002, 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.89153507300000001, 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "reached pg", 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.865 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95176 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917994+0000", 2024-09-13T14:59:14.866 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.917992+0000", 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.867 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918043+0000", 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1029999999999998e-05 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.868 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918064+0000", 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.1078e-05 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.869 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918073+0000", 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6060000000000003e-06 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921343+0000", 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.003269597 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.870 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.921356+0000", 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3698e-05 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809462+0000", 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88810571900000002 2024-09-13T14:59:14.871 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.809529+0000", 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7564999999999994e-05 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.872 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95188 9.6 9:6261a5c8:::benchmark_data_smithi017_339564_object95187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:59:14.873 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 484.943452304, 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.90364369099999997, 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95188 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.874 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918092+0000", 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:14.875 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918128+0000", 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6738e-05 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918136+0000", 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8599999999999993e-06 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.876 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918141+0000", 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6580000000000001e-06 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943565+0000", 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025424162 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943572+0000", 2024-09-13T14:59:14.877 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.985e-06 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808380+0000", 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86480780800000001 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808541+0000", 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000160958 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.878 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.808712+0000", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000170958 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.810740+0000", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002028555 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.813879+0000", 2024-09-13T14:59:14.879 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0031388950000000001 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821630+0000", 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0077507640000000003 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821692+0000", 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2106000000000005e-05 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.880 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.821736+0000", 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.3798999999999999e-05 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95192 9.16 9:6c6762a8:::benchmark_data_smithi017_339564_object95191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 484.94339708500002, 2024-09-13T14:59:14.881 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.917525484, 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95192 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.882 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.883 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.883 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918147+0000", 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.884 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918188+0000", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.0748999999999999e-05 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918195+0000", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.153e-06 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918199+0000", 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.194e-06 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.885 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943779+0000", 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025580471 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943784+0000", 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.0289999999999996e-06 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826118+0000", 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88233390700000003 2024-09-13T14:59:14.888 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826264+0000", 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014587500000000001 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.826408+0000", 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014380600000000001 2024-09-13T14:59:14.889 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828905+0000", 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.002496631 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.830686+0000", 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00178167 2024-09-13T14:59:14.890 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835583+0000", 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0048965609999999998 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835635+0000", 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.2522000000000003e-05 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.891 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835673+0000", 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7376999999999998e-05 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:95195 9.16 9:6f91244f:::benchmark_data_smithi017_339564_object95194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e383)", 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:59:14.892 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 484.94334049100001, 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.91756910000000003, 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 95195 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.893 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918204+0000", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918203+0000", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918232+0000", 2024-09-13T14:59:14.894 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8778000000000001e-05 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918239+0000", 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8190000000000003e-06 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.918243+0000", 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1069999999999998e-06 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.895 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943799+0000", 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.025555422000000001 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:09.943802+0000", 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.749e-06 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827614+0000", 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.88381141699999999 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.896 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827747+0000", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013306200000000001 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.827906+0000", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159474 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.828913+0000", 2024-09-13T14:59:14.897 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.001006369 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.831206+0000", 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0022932680000000002 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835706+0000", 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0045005210000000004 2024-09-13T14:59:14.898 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835745+0000", 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8803000000000001e-05 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:51:10.835773+0000", 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7801e-05 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.899 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179257 9.12 9:4ecfe978:::benchmark_data_smithi017_339564_object59784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.59729246400002, 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7357178289999999, 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179257 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.900 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264252+0000", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264251+0000", 2024-09-13T14:59:14.901 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264257+0000", 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.4439999999999996e-06 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264275+0000", 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.8155000000000002e-05 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.902 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264288+0000", 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2853e-05 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264350+0000", 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1716000000000003e-05 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.264366+0000", 2024-09-13T14:59:14.903 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5863999999999999e-05 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818721+0000", 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.554355614 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818727+0000", 2024-09-13T14:59:14.904 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.9699999999999996e-06 2024-09-13T14:59:14.905 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.905 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.905 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.905 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822922+0000", 2024-09-13T14:59:14.905 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0041942770000001 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822928+0000", 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6710000000000002e-06 2024-09-13T14:59:14.906 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.907 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.907 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.907 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935124+0000", 2024-09-13T14:59:14.907 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.112195982 2024-09-13T14:59:14.907 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935256+0000", 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013175400000000001 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.908 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.909 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.909 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.935638+0000", 2024-09-13T14:59:14.909 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000382023 2024-09-13T14:59:14.909 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.909 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943650+0000", 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0080123120000000006 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.910 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967274+0000", 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.023624030000000001 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.911 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.998994+0000", 2024-09-13T14:59:14.912 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.031719232999999999 2024-09-13T14:59:14.912 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.912 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.912 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.912 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999066+0000", 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.2358000000000001e-05 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.999970+0000", 2024-09-13T14:59:14.913 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00090380899999999999 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.914 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179272 9.12 9:4bdebc4a:::benchmark_data_smithi017_339564_object59799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.915 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:59:14.915 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.56457408799997, 2024-09-13T14:59:14.915 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7069958839999999, 2024-09-13T14:59:14.915 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.915 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.916 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.916 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179272 2024-09-13T14:59:14.916 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.917 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.918 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.918 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.918 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296970+0000", 2024-09-13T14:59:14.918 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.918 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296969+0000", 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.919 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.920 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.920 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296974+0000", 2024-09-13T14:59:14.920 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1889999999999998e-06 2024-09-13T14:59:14.920 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.920 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296984+0000", 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0131e-05 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.296990+0000", 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.981e-06 2024-09-13T14:59:14.921 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297030+0000", 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9907000000000003e-05 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.297036+0000", 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1199999999999999e-06 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.922 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818748+0000", 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.52171188899999998 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818754+0000", 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822950+0000", 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004196181 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.923 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822956+0000", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.6510000000000004e-06 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.937987+0000", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.115030368 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938146+0000", 2024-09-13T14:59:14.924 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000159703 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.938563+0000", 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041611300000000002 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946223+0000", 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.007660498 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.925 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967408+0000", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.021185239000000002 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003056+0000", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.035647757000000002 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003120+0000", 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3798000000000001e-05 2024-09-13T14:59:14.926 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.003966+0000", 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00084632100000000003 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179274 9.12 9:4d2975ed:::benchmark_data_smithi017_339564_object59801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.55844337500002, 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7113344069999998, 2024-09-13T14:59:14.927 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179274 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.928 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303101+0000", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303104+0000", 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6600000000000001e-06 2024-09-13T14:59:14.929 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303112+0000", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.5519999999999999e-06 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303117+0000", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7759999999999997e-06 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303166+0000", 2024-09-13T14:59:14.930 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.9438000000000003e-05 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.303173+0000", 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.3130000000000001e-06 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818774+0000", 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.51560055900000001 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.931 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818779+0000", 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.3959999999999997e-06 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822980+0000", 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004200381 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.822987+0000", 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9269999999999999e-06 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.932 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.940936+0000", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1179493199999999 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941074+0000", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00013840599999999999 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.941402+0000", 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032734600000000001 2024-09-13T14:59:14.933 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948293+0000", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068912189999999996 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967530+0000", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019237275000000002 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013579+0000", 2024-09-13T14:59:14.934 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046049113000000003 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.013672+0000", 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2820999999999998e-05 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014435+0000", 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000763406 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.935 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179351 9.12 9:48a19431:::benchmark_data_smithi017_339564_object59878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.41960559, 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5735510970000002, 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179351 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.936 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441939+0000", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441937+0000", 2024-09-13T14:59:14.937 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441945+0000", 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.763e-06 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.938 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441962+0000", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7119999999999999e-05 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.441967+0000", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.75e-06 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442014+0000", 2024-09-13T14:59:14.939 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6425999999999999e-05 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.442021+0000", 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.126e-06 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818801+0000", 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.37678042 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.940 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818807+0000", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8989999999999998e-06 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823009+0000", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0042012419999999 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823015+0000", 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.477e-06 2024-09-13T14:59:14.941 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943680+0000", 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1206647809999999 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.943812+0000", 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000132278 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.942 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.944158+0000", 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000345869 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951355+0000", 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071972260000000001 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967668+0000", 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016312454000000001 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.943 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014476+0000", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.046808529000000001 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.014559+0000", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2373999999999994e-05 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015490+0000", 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00093137299999999999 2024-09-13T14:59:14.944 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179385 9.12 9:4cb1548a:::benchmark_data_smithi017_339564_object59912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.36173865900003, 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5163587010000001, 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.945 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179385 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499806+0000", 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.946 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499805+0000", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499809+0000", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.5070000000000002e-06 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499825+0000", 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.5942000000000001e-05 2024-09-13T14:59:14.947 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499831+0000", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6790000000000002e-06 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499889+0000", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8090000000000001e-05 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.499897+0000", 2024-09-13T14:59:14.948 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.2649999999999996e-06 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818835+0000", 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.31893799299999998 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818873+0000", 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.8136000000000002e-05 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.949 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823068+0000", 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004194765 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823075+0000", 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.1899999999999998e-06 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946257+0000", 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1231814090000001 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.950 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946386+0000", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00012880199999999999 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.946701+0000", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000315719 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954875+0000", 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081736080000000006 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.951 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967813+0000", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012937664 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015557+0000", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.047744885000000001 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.015636+0000", 2024-09-13T14:59:14.952 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.8121000000000006e-05 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016164+0000", 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00052882700000000005 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179396 9.12 9:4f84ec82:::benchmark_data_smithi017_339564_object59923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:59:14.953 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.33358956199999, 2024-09-13T14:59:14.954 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4892925720000001, 2024-09-13T14:59:14.954 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179396 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.955 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527955+0000", 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527953+0000", 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.956 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527959+0000", 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2009999999999996e-06 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527974+0000", 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4151999999999999e-05 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.527979+0000", 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4029999999999998e-06 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.957 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528038+0000", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.8718000000000003e-05 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.528051+0000", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2835e-05 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818899+0000", 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.29084879699999999 2024-09-13T14:59:14.958 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818906+0000", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2369999999999997e-06 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823104+0000", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004197929 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823110+0000", 2024-09-13T14:59:14.959 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9099999999999999e-06 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948322+0000", 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.1252110909999999 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948445+0000", 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000123237 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.960 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.948787+0000", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00034259500000000001 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.954934+0000", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061466480000000002 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.967970+0000", 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013036443 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.961 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016200+0000", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.048229669000000003 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.016731+0000", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00053046400000000002 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017247+0000", 2024-09-13T14:59:14.962 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00051684400000000003 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:179450 9.12 9:494d5338:::benchmark_data_smithi017_339564_object59977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 383.04889755599999, 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2051270810000001, 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.963 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 179450 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812647+0000", 2024-09-13T14:59:14.964 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812644+0000", 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999976 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812653+0000", 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7709999999999995e-06 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.965 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812673+0000", 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.9281000000000001e-05 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812681+0000", 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5040000000000002e-06 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812744+0000", 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.2874999999999997e-05 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.966 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.812756+0000", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.2223e-05 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818929+0000", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0061728369999999996 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:51.818935+0000", 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0499999999999997e-06 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.967 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823287+0000", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.004352226 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:52.823294+0000", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.3570000000000003e-06 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951389+0000", 2024-09-13T14:59:14.968 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.12809574 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.951538+0000", 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00014834099999999999 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.952029+0000", 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00049101500000000003 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.969 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.955950+0000", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0039217050000000002 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:53.968099+0000", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.012148681999999999 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017285+0000", 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.049186344 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.970 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017360+0000", 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4738000000000002e-05 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:52:54.017774+0000", 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00041378699999999999 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.971 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236410 9.12 9:4f0cdf2c:::benchmark_data_smithi017_339564_object116937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.23722839300001, 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0220686269999999, 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236410 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.972 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624316+0000", 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624314+0000", 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999981 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.973 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624321+0000", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8889999999999996e-06 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624332+0000", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0993e-05 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624338+0000", 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.4650000000000002e-06 2024-09-13T14:59:14.974 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624435+0000", 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.6966999999999995e-05 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.624452+0000", 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.7719000000000001e-05 2024-09-13T14:59:14.975 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.617701+0000", 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99324826399999999 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.617790+0000", 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.9486000000000003e-05 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:14.976 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.618219+0000", 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00042904299999999999 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625381+0000", 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0071616340000000001 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646269+0000", 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.020888259999999999 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.977 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646325+0000", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6217000000000001e-05 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646360+0000", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4588000000000002e-05 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646385+0000", 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.4833000000000001e-05 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.978 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236419 9.12 9:48d9d9e2:::benchmark_data_smithi017_339564_object116946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.23376912399999, 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0186878989999999, 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236419 2024-09-13T14:59:14.979 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.980 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627775+0000", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627778+0000", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7699999999999999e-06 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627787+0000", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.5580000000000001e-06 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.981 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.627791+0000", 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.1420000000000003e-06 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.629765+0000", 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0019737880000000002 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.629773+0000", 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4489999999999999e-06 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.982 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620327+0000", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.99055368200000005 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620388+0000", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1173999999999994e-05 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.620620+0000", 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000231506 2024-09-13T14:59:14.983 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627047+0000", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0064277859999999996 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646297+0000", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019249596000000001 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646417+0000", 2024-09-13T14:59:14.984 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000119707 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646443+0000", 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.6469999999999999e-05 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646463+0000", 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0013999999999998e-05 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.985 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236481 9.12 9:4a8c469b:::benchmark_data_smithi017_339564_object117008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.16759932799999, 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.95271523700000005, 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236481 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.986 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.987 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693945+0000", 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693947+0000", 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.7669999999999999e-06 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693968+0000", 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.0607000000000001e-05 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.988 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.693972+0000", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.755e-06 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.694008+0000", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.5969000000000001e-05 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.694013+0000", 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6550000000000001e-06 2024-09-13T14:59:14.989 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.621921+0000", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.92790775199999997 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.621992+0000", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.0696000000000002e-05 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.622313+0000", 2024-09-13T14:59:14.990 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032082699999999998 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627054+0000", 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0047417249999999996 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646496+0000", 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.019442073000000001 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.991 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646606+0000", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000109664 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646630+0000", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3507999999999999e-05 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646660+0000", 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.0679000000000002e-05 2024-09-13T14:59:14.992 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236525 9.12 9:49746eac:::benchmark_data_smithi017_339564_object117052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.12412882699999, 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.90935570600000004, 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:14.993 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236525 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737416+0000", 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.994 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737415+0000", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737422+0000", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.498e-06 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737433+0000", 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0361e-05 2024-09-13T14:59:14.995 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.737438+0000", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.1499999999999998e-06 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.744319+0000", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0068809559999999997 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.744327+0000", 2024-09-13T14:59:14.996 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.7329999999999992e-06 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.623594+0000", 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.87926612400000004 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.623681+0000", 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.721e-05 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:14.997 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.624015+0000", 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00033437000000000001 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630579+0000", 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0065640789999999996 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646523+0000", 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.015943969999999998 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.998 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646698+0000", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.000174659 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646737+0000", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.9223e-05 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646771+0000", 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.4217999999999997e-05 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:14.999 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236567 9.12 9:4f71d45d:::benchmark_data_smithi017_339564_object117094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.07493776600001, 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.86031424400000001, 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:15.000 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236567 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786607+0000", 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.001 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786606+0000", 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786611+0000", 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6069999999999998e-06 2024-09-13T14:59:15.002 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786626+0000", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.4457999999999999e-05 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786632+0000", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.0939999999999996e-06 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786707+0000", 2024-09-13T14:59:15.003 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.4763000000000002e-05 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.786716+0000", 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.2520000000000005e-06 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625443+0000", 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83872685999999996 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625523+0000", 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.9893000000000002e-05 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.625780+0000", 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00025712100000000002 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.635877+0000", 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010096905 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.005 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646804+0000", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.010927004000000001 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646846+0000", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.2595000000000001e-05 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646893+0000", 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.6697999999999999e-05 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.006 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646921+0000", 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.8022e-05 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236603 9.12 9:4a904420:::benchmark_data_smithi017_339564_object117130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.03229919299997, 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.83652915900000002, 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:15.007 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236603 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.008 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829245+0000", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829244+0000", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829249+0000", 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.8359999999999999e-06 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.009 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829258+0000", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.4889999999999995e-06 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829261+0000", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.636e-06 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829299+0000", 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.7959e-05 2024-09-13T14:59:15.010 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.829306+0000", 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.9249999999999998e-06 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.011 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.011 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:14 smithi189 ceph-mon[90992]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627087+0000", 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.797780716 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627155+0000", 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.7937999999999999e-05 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.012 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.627475+0000", 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032002899999999999 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.635884+0000", 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0084087950000000002 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.013 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.646982+0000", 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.011098498 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665713+0000", 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.018730268000000001 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665749+0000", 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.6667e-05 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.665774+0000", 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5150000000000001e-05 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236628 9.12 9:48d6fa8d:::benchmark_data_smithi017_339564_object117155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 319.00143955300001, 2024-09-13T14:59:15.015 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.80943090500000003, 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236628 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:59:15.016 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860105+0000", 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860104+0000", 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.017 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860110+0000", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.8539999999999999e-06 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860124+0000", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.3287e-05 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860129+0000", 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 5.6590000000000004e-06 2024-09-13T14:59:15.018 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860191+0000", 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.1199999999999997e-05 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.860200+0000", 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.149e-06 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.019 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.628814+0000", 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.76861446499999997 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.628907+0000", 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3015000000000003e-05 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.629214+0000", 2024-09-13T14:59:15.020 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00030697 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639163+0000", 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0099489750000000005 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.647021+0000", 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078578490000000001 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.021 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669475+0000", 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022453923000000001 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669506+0000", 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 3.1111000000000001e-05 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669536+0000", 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.9697999999999999e-05 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.022 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236642 9.12 9:48b908b1:::benchmark_data_smithi017_339564_object117169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 318.98976865600002, 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.79782054800000002, 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236642 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.023 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871776+0000", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871775+0000", 2024-09-13T14:59:15.024 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871779+0000", 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.7339999999999999e-06 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871790+0000", 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0104e-05 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.025 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.871796+0000", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.392e-06 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.872081+0000", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00028549499999999998 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.872091+0000", 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 9.3980000000000005e-06 2024-09-13T14:59:15.026 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630615+0000", 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.75852406299999997 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.630680+0000", 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.5501999999999997e-05 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:15.027 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.631003+0000", 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00032254099999999997 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639191+0000", 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0081879980000000002 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.647059+0000", 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.0078676460000000007 2024-09-13T14:59:15.028 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669556+0000", 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.022497663000000001 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669579+0000", 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.3068e-05 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:15.029 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669596+0000", 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.6988999999999999e-05 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "description": "osd_op(client.46712.0:236688 9.12 9:4be6dfa1:::benchmark_data_smithi017_339564_object117215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "initiated_at": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "age": 318.93269051599998, 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.74087705599999998, 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "type_data": { 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:15.030 INFO:teuthology.orchestra.run.smithi189.stdout: "client_info": { 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "client": "client.46712", 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "client_addr": "172.21.15.17:0/606501292", 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "tid": 236688 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "events": [ 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "initiated", 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.031 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "throttled", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928854+0000", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "header_read", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928853+0000", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4294967295.999999 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "all_read", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928858+0000", 2024-09-13T14:59:15.032 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 4.826e-06 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "dispatched", 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928868+0000", 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0648e-05 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "queued_for_pg", 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928875+0000", 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 6.263e-06 2024-09-13T14:59:15.033 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928951+0000", 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 7.6817000000000007e-05 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for peered", 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:55.928962+0000", 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 1.0577999999999999e-05 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.034 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "reached_pg", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632359+0000", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.70339721499999996 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "started", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632445+0000", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 8.6211999999999994e-05 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.632673+0000", 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.00022758300000000001 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.035 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "op_commit", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.639196+0000", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.006522506 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.653018+0000", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.013822723 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669682+0000", 2024-09-13T14:59:15.036 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 0.016663846 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "commit_sent", 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669705+0000", 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.2875e-05 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: }, 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: { 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "event": "done", 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "time": "2024-09-13T14:53:56.669731+0000", 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: "duration": 2.5918000000000002e-05 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.037 INFO:teuthology.orchestra.run.smithi189.stdout: } 2024-09-13T14:59:15.038 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-09-13T14:59:15.038 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-09-13T14:59:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:14 smithi017 ceph-mon[103001]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:14 smithi017 ceph-mon[107556]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:15.896 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-13T14:59:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:15 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:15 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:59:16.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:15 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:15 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:16.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:59:17.012 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:17.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:17.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:17.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:59:17.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:17.133 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:17.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2604049644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:17.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T14:59:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:17] "GET /metrics HTTP/1.1" 200 34304 "" "Prometheus/2.51.0" 2024-09-13T14:59:18.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:17 smithi017 ceph-mon[103001]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:17 smithi017 ceph-mon[107556]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:18.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:17 smithi189 ceph-mon[90992]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:19.139 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:59:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:19 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1335634767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:19 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:19 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1335634767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:19 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:19 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1335634767' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:19 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:19.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-13T14:59:20.117 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:20.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:20.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[103001]: osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:59:20.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[103001]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[107556]: osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:59:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:20 smithi017 ceph-mon[107556]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:20 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:20 smithi189 ceph-mon[90992]: osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:59:20.506 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:20 smithi189 ceph-mon[90992]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:22.564 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-13T14:59:22.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[107556]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:22.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/287999816' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:22.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:22.920 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[103001]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:22.920 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/287999816' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:22.920 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:22 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:22 smithi189 ceph-mon[90992]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:22 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/287999816' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:22 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:59:23.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:59:23.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:23 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:23.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:23 smithi017 ceph-mon[103001]: osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:59:23.608 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:23.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:23 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:23.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:23 smithi017 ceph-mon[107556]: osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:59:24.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:23 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:59:24.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:23 smithi189 ceph-mon[90992]: osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:59:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:24 smithi017 ceph-mon[103001]: pgmap v1662: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:24 smithi017 ceph-mon[103001]: osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:59:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:24 smithi017 ceph-mon[107556]: pgmap v1662: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:24 smithi017 ceph-mon[107556]: osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:59:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:24 smithi189 ceph-mon[90992]: pgmap v1662: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:24 smithi189 ceph-mon[90992]: osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:59:25.579 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:59:25.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:25 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1334999650' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:25.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:25 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:25.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:25 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1334999650' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:25.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:25 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:25 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1334999650' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:26.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:25 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:26.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-13T14:59:26.486 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:26.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[103001]: pgmap v1664: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:26.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:26.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[103001]: osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:59:26.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[107556]: pgmap v1664: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:26.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:26.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:26 smithi017 ceph-mon[107556]: osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:59:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:26 smithi189 ceph-mon[90992]: pgmap v1664: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:26 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:26 smithi189 ceph-mon[90992]: osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:59:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:27.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:27] "GET /metrics HTTP/1.1" 200 34301 "" "Prometheus/2.51.0" 2024-09-13T14:59:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:28.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:28.584 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-13T14:59:28.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:28 smithi017 ceph-mon[103001]: pgmap v1666: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:28.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:28 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:28.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:28 smithi017 ceph-mon[107556]: pgmap v1666: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:28 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:29.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:28 smithi189 ceph-mon[90992]: pgmap v1666: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:29.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:28 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:59:29.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:59:29.513 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:29 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:29 smithi017 ceph-mon[107556]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:59:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:29 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:29 smithi017 ceph-mon[103001]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:59:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:29 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2187721509' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:59:30.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:29 smithi189 ceph-mon[90992]: osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:59:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:30.783 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:30 smithi189 ceph-mon[90992]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:30.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:30 smithi017 ceph-mon[107556]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:30.818 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:30 smithi017 ceph-mon[103001]: pgmap v1668: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:31.600 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:59:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:31 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:31 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:32.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:31 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:32.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-13T14:59:32.619 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:32.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[103001]: pgmap v1669: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:32.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:32.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[103001]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:59:32.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[107556]: pgmap v1669: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:32.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:32.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:32 smithi017 ceph-mon[107556]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:59:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:32 smithi189 ceph-mon[90992]: pgmap v1669: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:32 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3171312668' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:32 smithi189 ceph-mon[90992]: osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:59:34.605 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-13T14:59:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:34 smithi017 ceph-mon[103001]: pgmap v1671: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:34 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:34 smithi017 ceph-mon[107556]: pgmap v1671: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:34 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:34 smithi189 ceph-mon[90992]: pgmap v1671: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:34 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:59:35.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:59:35.622 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:35.870 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:35.870 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:35 smithi017 ceph-mon[103001]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:59:35.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:35.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:35 smithi017 ceph-mon[107556]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:59:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3767636832' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:59:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:35 smithi189 ceph-mon[90992]: osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:59:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:36 smithi017 ceph-mon[103001]: pgmap v1673: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:36 smithi017 ceph-mon[107556]: pgmap v1673: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:37.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:36 smithi189 ceph-mon[90992]: pgmap v1673: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:37.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:37] "GET /metrics HTTP/1.1" 200 34301 "" "Prometheus/2.51.0" 2024-09-13T14:59:37.623 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:59:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:37 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:37 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:38.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:37 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:38.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-13T14:59:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[103001]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[103001]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:59:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[107556]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:38 smithi017 ceph-mon[107556]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:59:38.702 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:39.005 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:38 smithi189 ceph-mon[90992]: pgmap v1674: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1215428226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:38 smithi189 ceph-mon[90992]: osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:59:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:40 smithi017 ceph-mon[103001]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:40 smithi017 ceph-mon[107556]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:40 smithi189 ceph-mon[90992]: pgmap v1676: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:41.633 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-13T14:59:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:42.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:59:42.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:59:42.641 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[103001]: pgmap v1677: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[103001]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[107556]: pgmap v1677: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[107556]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:59:42.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:42 smithi189 ceph-mon[90992]: pgmap v1677: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:42 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/177441782' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:59:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:42 smithi189 ceph-mon[90992]: osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:59:43.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:44 smithi017 ceph-mon[103001]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:44.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:44 smithi017 ceph-mon[107556]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:44 smithi189 ceph-mon[90992]: pgmap v1679: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:45.650 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:59:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/809831057' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:45 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/809831057' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:45 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/809831057' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:46.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:45 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:46.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-13T14:59:46.621 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:46.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[103001]: pgmap v1680: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:46.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:46.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[103001]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:59:46.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[107556]: pgmap v1680: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:46.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:46.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:46 smithi017 ceph-mon[107556]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:59:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:46 smithi189 ceph-mon[90992]: pgmap v1680: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:46 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:47.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:46 smithi189 ceph-mon[90992]: osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:59:47.508 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:47] "GET /metrics HTTP/1.1" 200 34302 "" "Prometheus/2.51.0" 2024-09-13T14:59:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:48 smithi017 ceph-mon[103001]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:48 smithi017 ceph-mon[107556]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:49.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:48 smithi189 ceph-mon[90992]: pgmap v1682: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:49.661 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-13T14:59:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/208485564' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:49 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/208485564' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:49 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/208485564' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:50.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:49 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:59:50.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:59:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 14:59:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T14:59:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[103001]: pgmap v1683: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[103001]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:59:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[107556]: pgmap v1683: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:50 smithi017 ceph-mon[107556]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:59:50.728 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:50 smithi189 ceph-mon[90992]: pgmap v1683: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:50 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:59:51.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:50 smithi189 ceph-mon[90992]: osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:59:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:52 smithi017 ceph-mon[103001]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:52 smithi017 ceph-mon[107556]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:52 smithi189 ceph-mon[90992]: pgmap v1685: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:53.667 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:59:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:53 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:53 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:54.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:53 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:54.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-13T14:59:54.771 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:54.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[103001]: pgmap v1686: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:54.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:54.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[103001]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:59:54.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[107556]: pgmap v1686: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:54.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:54.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:54 smithi017 ceph-mon[107556]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:59:55.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:54 smithi189 ceph-mon[90992]: pgmap v1686: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:55.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:54 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1859740682' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:55.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:54 smithi189 ceph-mon[90992]: osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:59:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:56 smithi017 ceph-mon[103001]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:56 smithi017 ceph-mon[107556]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:56 smithi189 ceph-mon[90992]: pgmap v1688: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:57.671 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:14:59:57] "GET /metrics HTTP/1.1" 200 34300 "" "Prometheus/2.51.0" 2024-09-13T14:59:57.679 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-13T14:59:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2709083302' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2709083302' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:57 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2709083302' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:57 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:59:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:58.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:59:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[107556]: pgmap v1689: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[107556]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[107556]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:59:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[103001]: pgmap v1689: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[103001]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 14:59:58 smithi017 ceph-mon[103001]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:59:58.749 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:58 smithi189 ceph-mon[90992]: pgmap v1689: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:59:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:58 smithi189 ceph-mon[90992]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:59:59.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 14:59:58 smithi189 ceph-mon[90992]: osdmap e516: 8 total, 8 up, 8 in 2024-09-13T15:00:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:00.681 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:00:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[103001]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[103001]: overall HEALTH_OK 2024-09-13T15:00:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[107556]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[107556]: overall HEALTH_OK 2024-09-13T15:00:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:00 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:01.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:00 smithi189 ceph-mon[90992]: pgmap v1691: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:01.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:00 smithi189 ceph-mon[90992]: overall HEALTH_OK 2024-09-13T15:00:01.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:00 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:01.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-13T15:00:01.744 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:01.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:01 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:01.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:01 smithi017 ceph-mon[103001]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T15:00:01.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:01 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:01.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:01 smithi017 ceph-mon[107556]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T15:00:02.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:01 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/271277374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:02.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:01 smithi189 ceph-mon[90992]: osdmap e517: 8 total, 8 up, 8 in 2024-09-13T15:00:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:02 smithi017 ceph-mon[103001]: pgmap v1693: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:02 smithi017 ceph-mon[107556]: pgmap v1693: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:03.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:02 smithi189 ceph-mon[90992]: pgmap v1693: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:03.697 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-13T15:00:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:03 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:03 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:04.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:03 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:00:04.299 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-13T15:00:04.299 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-13T15:00:04.299 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-13T15:00:04.300 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-13T15:00:04.300 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T15:00:04.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:00:04.559 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[107556]: pgmap v1694: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[107556]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T15:00:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[103001]: pgmap v1694: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:04 smithi017 ceph-mon[103001]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T15:00:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:04 smithi189 ceph-mon[90992]: pgmap v1694: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:04 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1611379364' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:00:05.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:04 smithi189 ceph-mon[90992]: osdmap e518: 8 total, 8 up, 8 in 2024-09-13T15:00:06.520 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T15:00:06.520 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":518,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T15:00:03.691306+0000","last_up_change":"2024-09-13T14:40:43.675619+0000","last_in_change":"2024-09-13T14:53:08.427118+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"499","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"487","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":492,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6803","nonce":59727078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6805","nonce":59727078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6809","nonce":59727078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6807","nonce":59727078}]},"public_addr":"172.21.15.17:6803/59727078","cluster_addr":"172.21.15.17:6805/59727078","heartbeat_back_addr":"172.21.15.17:6809/59727078","heartbeat_front_addr":"172.21.15.17:6807/59727078","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":91,"last_clean_end":234,"up_from":244,"up_thru":492,"down_at":235,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6811","nonce":176828329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6813","nonce":176828329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6817","nonce":176828329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6815","nonce":176828329}]},"public_addr":"172.21.15.17:6811/176828329","cluster_addr":"172.21.15.17:6813/176828329","heartbeat_back_addr":"172.21.15.17:6817/176828329","heartbeat_front_addr":"172.21.15.17:6815/176828329","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":92,"up_from":96,"up_thru":505,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6819","nonce":847801397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6821","nonce":847801397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6825","nonce":847801397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6823","nonce":847801397}]},"public_addr":"172.21.15.17:6819/847801397","cluster_addr":"172.21.15.17:6821/847801397","heartbeat_back_addr":"172.21.15.17:6825/847801397","heartbeat_front_addr":"172.21.15.17:6823/847801397","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":97,"up_from":101,"up_thru":492,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6827","nonce":594040290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6829","nonce":594040290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6833","nonce":594040290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6831","nonce":594040290}]},"public_addr":"172.21.15.17:6827/594040290","cluster_addr":"172.21.15.17:6829/594040290","heartbeat_back_addr":"172.21.15.17:6833/594040290","heartbeat_front_addr":"172.21.15.17:6831/594040290","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":294,"up_from":316,"up_thru":492,"down_at":295,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6809","nonce":1980893081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6811","nonce":1980893081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6815","nonce":1980893081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6813","nonce":1980893081}]},"public_addr":"172.21.15.189:6809/1980893081","cluster_addr":"172.21.15.189:6811/1980893081","heartbeat_back_addr":"172.21.15.189:6815/1980893081","heartbeat_front_addr":"172.21.15.189:6813/1980893081","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":234},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":92},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":97},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":294},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.189:0/2620105210":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.189:6835/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/1432283173":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3987946356":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/3334536375":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3205338252":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/2573022556":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/652586513":"2024-09-14T14:17:29.088581+0000","172.21.15.189:0/2946870083":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/1507399137":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/2755559292":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/3476899836":"2024-09-14T14:16:12.599650+0000","172.21.15.189:6834/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3929495183":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1515064414":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/282666625":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/4174048439":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6801/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6801/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/101070012":"2024-09-14T14:16:12.599650+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:00:06.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:06 smithi017 ceph-mon[103001]: pgmap v1696: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:06.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:06 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1202916895' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:06.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:06 smithi017 ceph-mon[107556]: pgmap v1696: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:06.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:06 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1202916895' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:06 smithi189 ceph-mon[90992]: pgmap v1696: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:07.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:06 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1202916895' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:07.248 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T15:00:07.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:00:07.506 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:07.628 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:07 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:07] "GET /metrics HTTP/1.1" 200 34300 "" "Prometheus/2.51.0" 2024-09-13T15:00:09.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:08 smithi189 ceph-mon[90992]: pgmap v1697: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:09.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:08 smithi017 ceph-mon[107556]: pgmap v1697: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:09.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:08 smithi017 ceph-mon[103001]: pgmap v1697: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:09.432 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T15:00:09.432 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":518,"fsid":"523f52e4-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:06:56.701743+0000","modified":"2024-09-13T15:00:03.691306+0000","last_up_change":"2024-09-13T14:40:43.675619+0000","last_in_change":"2024-09-13T14:53:08.427118+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:09:56.376395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"499","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"487","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:12:25.199564+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":78,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:13:39.659995+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:13:40.551728+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b9a98879-a576-4dbf-80ce-247ecc208028","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":492,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6803","nonce":59727078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6805","nonce":59727078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6809","nonce":59727078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":59727078},{"type":"v1","addr":"172.21.15.17:6807","nonce":59727078}]},"public_addr":"172.21.15.17:6803/59727078","cluster_addr":"172.21.15.17:6805/59727078","heartbeat_back_addr":"172.21.15.17:6809/59727078","heartbeat_front_addr":"172.21.15.17:6807/59727078","state":["exists","up"]},{"osd":1,"uuid":"f7dc93a9-e0e0-4e5a-a56f-f9bad21089b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":91,"last_clean_end":234,"up_from":244,"up_thru":492,"down_at":235,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6811","nonce":176828329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6813","nonce":176828329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6817","nonce":176828329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":176828329},{"type":"v1","addr":"172.21.15.17:6815","nonce":176828329}]},"public_addr":"172.21.15.17:6811/176828329","cluster_addr":"172.21.15.17:6813/176828329","heartbeat_back_addr":"172.21.15.17:6817/176828329","heartbeat_front_addr":"172.21.15.17:6815/176828329","state":["exists","up"]},{"osd":2,"uuid":"eb11f41d-35bc-4180-a0f6-781c7dbe5b5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":92,"up_from":96,"up_thru":505,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6819","nonce":847801397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6821","nonce":847801397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6825","nonce":847801397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":847801397},{"type":"v1","addr":"172.21.15.17:6823","nonce":847801397}]},"public_addr":"172.21.15.17:6819/847801397","cluster_addr":"172.21.15.17:6821/847801397","heartbeat_back_addr":"172.21.15.17:6825/847801397","heartbeat_front_addr":"172.21.15.17:6823/847801397","state":["exists","up"]},{"osd":3,"uuid":"76cf0d9c-0405-4601-9c2c-3cbbba55dab8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":97,"up_from":101,"up_thru":492,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6827","nonce":594040290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6829","nonce":594040290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6833","nonce":594040290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":594040290},{"type":"v1","addr":"172.21.15.17:6831","nonce":594040290}]},"public_addr":"172.21.15.17:6827/594040290","cluster_addr":"172.21.15.17:6829/594040290","heartbeat_back_addr":"172.21.15.17:6833/594040290","heartbeat_front_addr":"172.21.15.17:6831/594040290","state":["exists","up"]},{"osd":4,"uuid":"698eddb3-30d9-431e-9731-9c37441f4359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6801","nonce":3136403996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6803","nonce":3136403996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6807","nonce":3136403996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3136403996},{"type":"v1","addr":"172.21.15.189:6805","nonce":3136403996}]},"public_addr":"172.21.15.189:6801/3136403996","cluster_addr":"172.21.15.189:6803/3136403996","heartbeat_back_addr":"172.21.15.189:6807/3136403996","heartbeat_front_addr":"172.21.15.189:6805/3136403996","state":["exists","up"]},{"osd":5,"uuid":"60f030c1-c222-4a1e-b7d6-d788f37bc5eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":37,"last_clean_end":294,"up_from":316,"up_thru":492,"down_at":295,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6809","nonce":1980893081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6811","nonce":1980893081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6815","nonce":1980893081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":1980893081},{"type":"v1","addr":"172.21.15.189:6813","nonce":1980893081}]},"public_addr":"172.21.15.189:6809/1980893081","cluster_addr":"172.21.15.189:6811/1980893081","heartbeat_back_addr":"172.21.15.189:6815/1980893081","heartbeat_front_addr":"172.21.15.189:6813/1980893081","state":["exists","up"]},{"osd":6,"uuid":"a53bc1f8-35e0-4e97-a9d1-90b318be2165","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6817","nonce":2742951256}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6819","nonce":2742951256}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6823","nonce":2742951256}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":2742951256},{"type":"v1","addr":"172.21.15.189:6821","nonce":2742951256}]},"public_addr":"172.21.15.189:6817/2742951256","cluster_addr":"172.21.15.189:6819/2742951256","heartbeat_back_addr":"172.21.15.189:6823/2742951256","heartbeat_front_addr":"172.21.15.189:6821/2742951256","state":["exists","up"]},{"osd":7,"uuid":"a3fdb8ae-2c63-493d-b2ac-8009b024fa12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":492,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6825","nonce":1790039107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6827","nonce":1790039107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6831","nonce":1790039107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":1790039107},{"type":"v1","addr":"172.21.15.189:6829","nonce":1790039107}]},"public_addr":"172.21.15.189:6825/1790039107","cluster_addr":"172.21.15.189:6827/1790039107","heartbeat_back_addr":"172.21.15.189:6831/1790039107","heartbeat_front_addr":"172.21.15.189:6829/1790039107","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:06.199515+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:29.664378+0000","dead_epoch":234},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:09:52.716006+0000","dead_epoch":92},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:17.917918+0000","dead_epoch":97},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:10:39.497860+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:03.558444+0000","dead_epoch":294},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:27.437768+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:11:52.166643+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1928248055":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/3701412343":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/580144606":"2024-09-14T14:07:45.879701+0000","172.21.15.189:0/2620105210":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1221135323":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6801/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.189:6835/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/1432283173":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3987946356":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/3469824274":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/3334536375":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3205338252":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/2573022556":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/652586513":"2024-09-14T14:17:29.088581+0000","172.21.15.189:0/2946870083":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6800/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/4035711494":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3527746617":"2024-09-14T14:12:23.277718+0000","172.21.15.189:0/1507399137":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/3892697089":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/2755559292":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/3476899836":"2024-09-14T14:16:12.599650+0000","172.21.15.189:6834/862123899":"2024-09-14T14:17:29.088581+0000","172.21.15.17:0/1490019363":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:6800/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6801/2902550950":"2024-09-14T14:12:23.277718+0000","172.21.15.17:0/3173583319":"2024-09-14T14:12:23.277718+0000","172.21.15.17:6801/502430602":"2024-09-14T14:07:45.879701+0000","172.21.15.17:0/3929495183":"2024-09-14T14:18:11.614839+0000","172.21.15.17:6800/2576928215":"2024-09-14T14:07:23.765308+0000","172.21.15.17:0/1515064414":"2024-09-14T14:16:12.599650+0000","172.21.15.17:0/282666625":"2024-09-14T14:18:11.614839+0000","172.21.15.189:0/4174048439":"2024-09-14T14:17:29.088581+0000","172.21.15.17:6801/2742834835":"2024-09-14T14:16:12.599650+0000","172.21.15.17:6801/3409877178":"2024-09-14T14:18:11.614839+0000","172.21.15.17:0/101070012":"2024-09-14T14:16:12.599650+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:00:10.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:09 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3637283395' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:10.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:09 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3637283395' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:10.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:09 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3637283395' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:00:10.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T15:00:10.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T15:00:10.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T15:00:10.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T15:00:10.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T15:00:10.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T15:00:10.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T15:00:10.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T15:00:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:10.650 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.655 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.655 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.664 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:10.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:11.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:10 smithi189 ceph-mon[90992]: pgmap v1698: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:11.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:10 smithi017 ceph-mon[103001]: pgmap v1698: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:11.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:10 smithi017 ceph-mon[107556]: pgmap v1698: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:12.770 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:12 smithi017 ceph-mon[103001]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:12.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:12 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:12.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:12 smithi017 ceph-mon[107556]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:12.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:12 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:12 smithi189 ceph-mon[90992]: pgmap v1699: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:13.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:12 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:14 smithi017 ceph-mon[103001]: pgmap v1700: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:14 smithi017 ceph-mon[107556]: pgmap v1700: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:15.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:14 smithi189 ceph-mon[90992]: pgmap v1700: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:16 smithi017 ceph-mon[107556]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:16 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:16 smithi017 ceph-mon[103001]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:16 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:16 smithi189 ceph-mon[90992]: pgmap v1701: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:17.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:16 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:17.563 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:17 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:17] "GET /metrics HTTP/1.1" 200 34301 "" "Prometheus/2.51.0" 2024-09-13T15:00:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:18 smithi017 ceph-mon[103001]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:18 smithi017 ceph-mon[107556]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:19.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:18 smithi189 ceph-mon[90992]: pgmap v1702: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:19.386 INFO:teuthology.orchestra.run.smithi017.stdout:184683594318 2024-09-13T15:00:19.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T15:00:20.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:20 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:20.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[103001]: pgmap v1703: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:20.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:20.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:20.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[107556]: pgmap v1703: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:20.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:20.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:20 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:20 smithi189 ceph-mon[90992]: pgmap v1703: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:21.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:20 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:412316860861 2024-09-13T15:00:22.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T15:00:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[103001]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[107556]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:22 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:22 smithi189 ceph-mon[90992]: pgmap v1704: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:23.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:22 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:24.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:24 smithi017 ceph-mon[103001]: pgmap v1705: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:24 smithi017 ceph-mon[107556]: pgmap v1705: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:25.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:24 smithi189 ceph-mon[90992]: pgmap v1705: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:25.968 INFO:teuthology.orchestra.run.smithi017.stdout:369367187924 2024-09-13T15:00:25.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T15:00:26.666 INFO:teuthology.orchestra.run.smithi017.stdout:1357209665773 2024-09-13T15:00:26.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T15:00:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:26 smithi017 ceph-mon[103001]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:26 smithi017 ceph-mon[107556]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:27.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:26 smithi189 ceph-mon[90992]: pgmap v1706: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:27.125 INFO:teuthology.orchestra.run.smithi017.stdout:433791697336 2024-09-13T15:00:27.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T15:00:27.604 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:27 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:27] "GET /metrics HTTP/1.1" 200 34302 "" "Prometheus/2.51.0" 2024-09-13T15:00:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:27 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:27 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:28.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:27 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:28.085 INFO:teuthology.orchestra.run.smithi017.stdout:1047972020533 2024-09-13T15:00:28.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T15:00:28.923 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:28 smithi017 ceph-mon[103001]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:28.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:28 smithi017 ceph-mon[107556]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:29.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:28 smithi189 ceph-mon[90992]: pgmap v1707: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:29.341 INFO:teuthology.orchestra.run.smithi017.stdout:133143986777 2024-09-13T15:00:29.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T15:00:29.891 INFO:teuthology.orchestra.run.smithi017.stdout:210453398091 2024-09-13T15:00:29.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T15:00:29.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:29.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:29.930 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:30.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:30 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:30.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:30 smithi017 ceph-mon[107556]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:30.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:30 smithi017 ceph-mon[103001]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:31.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:30 smithi189 ceph-mon[90992]: pgmap v1708: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:32.894 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:32 smithi017 ceph-mon[103001]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:32.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:32 smithi017 ceph-mon[107556]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:33.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:32 smithi189 ceph-mon[90992]: pgmap v1709: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:34.087 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:34.087 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:34.089 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:34 smithi017 ceph-mon[103001]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:34 smithi017 ceph-mon[107556]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:35.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:34 smithi189 ceph-mon[90992]: pgmap v1710: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:35.209 INFO:teuthology.orchestra.run.smithi017.stdout:412316860864 2024-09-13T15:00:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:35 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/3465006720' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:00:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:35 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/3465006720' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:00:36.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:35 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/3465006720' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:00:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:36 smithi017 ceph-mon[103001]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:36 smithi017 ceph-mon[107556]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:36.989 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860861 got 412316860864 for osd.2 2024-09-13T15:00:36.989 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:37.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:36 smithi189 ceph-mon[90992]: pgmap v1711: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:37.476 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:37 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:37] "GET /metrics HTTP/1.1" 200 34302 "" "Prometheus/2.51.0" 2024-09-13T15:00:38.510 INFO:teuthology.orchestra.run.smithi017.stdout:369367187926 2024-09-13T15:00:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:38 smithi189 ceph-mon[90992]: pgmap v1712: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:39.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1385207862' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:00:39.007 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:38 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/542940657' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[107556]: pgmap v1712: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1385207862' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/542940657' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[103001]: pgmap v1712: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1385207862' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:00:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:38 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/542940657' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:00:39.350 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367187924 got 369367187926 for osd.0 2024-09-13T15:00:39.351 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:39.841 INFO:teuthology.orchestra.run.smithi017.stdout:184683594322 2024-09-13T15:00:40.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:41.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:40 smithi189 ceph-mon[90992]: pgmap v1713: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:41.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:40 smithi017 ceph-mon[107556]: pgmap v1713: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:41.044 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:40 smithi017 ceph-mon[103001]: pgmap v1713: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:41.343 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:41.344 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:41 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2288745845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:00:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:41 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2288745845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:00:42.217 INFO:teuthology.orchestra.run.smithi017.stdout:1047972020536 2024-09-13T15:00:42.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:41 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2288745845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:00:42.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:42 smithi017 ceph-mon[103001]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:42.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:42 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:42.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:42 smithi017 ceph-mon[107556]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:42.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:42 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:42.948 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1047972020533 got 1047972020536 for osd.1 2024-09-13T15:00:42.948 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:43.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:42 smithi189 ceph-mon[90992]: pgmap v1714: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:43.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:42 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:43.526 INFO:teuthology.orchestra.run.smithi017.stdout:133143986779 2024-09-13T15:00:43.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:43 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/2027075851' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:00:43.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:43 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/2027075851' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:00:44.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:43 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/2027075851' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:00:44.866 INFO:teuthology.orchestra.run.smithi017.stdout:1357209665777 2024-09-13T15:00:44.867 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:44 smithi017 ceph-mon[103001]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:44.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:44 smithi017 ceph-mon[107556]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:45.215 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594318 got 184683594322 for osd.6 2024-09-13T15:00:45.215 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:45.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:44 smithi189 ceph-mon[90992]: pgmap v1715: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:45.716 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986777 got 133143986779 for osd.4 2024-09-13T15:00:45.717 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:45 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/734692777' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:00:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:45 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/734692777' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:00:46.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:45 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/734692777' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:00:46.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:46 smithi017 ceph-mon[107556]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:46.986 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:46 smithi017 ceph-mon[103001]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:47.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:46 smithi189 ceph-mon[90992]: pgmap v1716: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:47.530 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1357209665773 got 1357209665777 for osd.5 2024-09-13T15:00:47.530 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:47.623 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:47 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:47] "GET /metrics HTTP/1.1" 200 34301 "" "Prometheus/2.51.0" 2024-09-13T15:00:47.917 INFO:teuthology.orchestra.run.smithi017.stdout:433791697340 2024-09-13T15:00:48.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:48 smithi017 ceph-mon[103001]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:48.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:48 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/1639266673' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:00:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:48 smithi017 ceph-mon[107556]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:48 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/1639266673' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:00:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:48 smithi189 ceph-mon[90992]: pgmap v1717: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:49.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:48 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/1639266673' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:00:49.331 INFO:tasks.cephadm.ceph_manager.ceph:need seq 433791697336 got 433791697340 for osd.3 2024-09-13T15:00:49.332 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:49.694 INFO:teuthology.orchestra.run.smithi017.stdout:210453398096 2024-09-13T15:00:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:49 smithi017 ceph-mon[103001]: from='client.? 172.21.15.17:0/487105018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:00:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:49 smithi017 ceph-mon[107556]: from='client.? 172.21.15.17:0/487105018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:00:50.213 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:49 smithi189 ceph-mon[90992]: from='client.? 172.21.15.17:0/487105018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:00:50.421 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453398091 got 210453398096 for osd.7 2024-09-13T15:00:50.422 DEBUG:teuthology.parallel:result is None 2024-09-13T15:00:50.422 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:00:50.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:00:50.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:00:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:00:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:50 smithi017 ceph-mon[103001]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:50 smithi017 ceph-mon[107556]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:51.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:50 smithi189 ceph-mon[90992]: pgmap v1718: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:51.672 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:52.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:52 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: pgmap v1719: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:53.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:52 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' 2024-09-13T15:00:53.745 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T15:00:53.746 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T15:00:54.385 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1720,"stamp":"2024-09-13T15:00:53.527826+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":907,"num_write_kb":12623,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":8394258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4132,"ondisk_log_size":4132,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7423556,"kb_used_data":67108,"kb_used_omap":44,"kb_used_meta":7356307,"kb_avail":742537660,"statfs":{"total":767960285184,"available":760358563840,"internally_reserved":0,"allocated":68718592,"data_stored":47312999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7532858615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003136"},"pg_stats":[{"pgid":"3.1f","version":"64'4","reported_seq":219,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865436+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T15:00:04.865436+0000","last_peered":"2024-09-13T15:00:04.865436+0000","last_clean":"2024-09-13T15:00:04.865436+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T15:00:04.865436+0000","last_undegraded":"2024-09-13T15:00:04.865436+0000","last_fullsized":"2024-09-13T15:00:04.865436+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864588+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T15:00:00.864588+0000","last_peered":"2024-09-13T15:00:00.864588+0000","last_clean":"2024-09-13T15:00:00.864588+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T15:00:00.864588+0000","last_undegraded":"2024-09-13T15:00:00.864588+0000","last_fullsized":"2024-09-13T15:00:00.864588+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":77,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698373+0000","last_change":"2024-09-13T14:58:37.655579+0000","last_active":"2024-09-13T15:00:03.698373+0000","last_peered":"2024-09-13T15:00:03.698373+0000","last_clean":"2024-09-13T15:00:03.698373+0000","last_became_active":"2024-09-13T14:58:37.386557+0000","last_became_peered":"2024-09-13T14:58:37.386557+0000","last_unstale":"2024-09-13T15:00:03.698373+0000","last_undegraded":"2024-09-13T15:00:03.698373+0000","last_fullsized":"2024-09-13T15:00:03.698373+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:37.655535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:37.655535+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:15.463158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":924,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866767+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T15:00:04.866767+0000","last_peered":"2024-09-13T15:00:04.866767+0000","last_clean":"2024-09-13T15:00:04.866767+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T15:00:04.866767+0000","last_undegraded":"2024-09-13T15:00:04.866767+0000","last_fullsized":"2024-09-13T15:00:04.866767+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":950,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866352+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T15:00:04.866352+0000","last_peered":"2024-09-13T15:00:04.866352+0000","last_clean":"2024-09-13T15:00:04.866352+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T15:00:04.866352+0000","last_undegraded":"2024-09-13T15:00:04.866352+0000","last_fullsized":"2024-09-13T15:00:04.866352+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759021+0000","last_change":"2024-09-13T14:58:51.463423+0000","last_active":"2024-09-13T15:00:00.759021+0000","last_peered":"2024-09-13T15:00:00.759021+0000","last_clean":"2024-09-13T15:00:00.759021+0000","last_became_active":"2024-09-13T14:58:37.387887+0000","last_became_peered":"2024-09-13T14:58:37.387887+0000","last_unstale":"2024-09-13T15:00:00.759021+0000","last_undegraded":"2024-09-13T15:00:00.759021+0000","last_fullsized":"2024-09-13T15:00:00.759021+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:51.463355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:51.463355+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:45.169469+0000","scrub_duration":0.0018833960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":449,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683402+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T15:00:00.683402+0000","last_peered":"2024-09-13T15:00:00.683402+0000","last_clean":"2024-09-13T15:00:00.683402+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T15:00:00.683402+0000","last_undegraded":"2024-09-13T15:00:00.683402+0000","last_fullsized":"2024-09-13T15:00:00.683402+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":235,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864865+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T15:00:00.864865+0000","last_peered":"2024-09-13T15:00:00.864865+0000","last_clean":"2024-09-13T15:00:00.864865+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T15:00:00.864865+0000","last_undegraded":"2024-09-13T15:00:00.864865+0000","last_fullsized":"2024-09-13T15:00:00.864865+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683145+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T15:00:00.683145+0000","last_peered":"2024-09-13T15:00:00.683145+0000","last_clean":"2024-09-13T15:00:00.683145+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T15:00:00.683145+0000","last_undegraded":"2024-09-13T15:00:00.683145+0000","last_fullsized":"2024-09-13T15:00:00.683145+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864973+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T15:00:00.864973+0000","last_peered":"2024-09-13T15:00:00.864973+0000","last_clean":"2024-09-13T15:00:00.864973+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T15:00:00.864973+0000","last_undegraded":"2024-09-13T15:00:00.864973+0000","last_fullsized":"2024-09-13T15:00:00.864973+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":226,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866761+0000","last_change":"2024-09-13T14:58:37.837560+0000","last_active":"2024-09-13T15:00:04.866761+0000","last_peered":"2024-09-13T15:00:04.866761+0000","last_clean":"2024-09-13T15:00:04.866761+0000","last_became_active":"2024-09-13T14:58:37.385007+0000","last_became_peered":"2024-09-13T14:58:37.385007+0000","last_unstale":"2024-09-13T15:00:04.866761+0000","last_undegraded":"2024-09-13T15:00:04.866761+0000","last_fullsized":"2024-09-13T15:00:04.866761+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:37.837514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:37.837514+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:33:39.369516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866156+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T15:00:04.866156+0000","last_peered":"2024-09-13T15:00:04.866156+0000","last_clean":"2024-09-13T15:00:04.866156+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T15:00:04.866156+0000","last_undegraded":"2024-09-13T15:00:04.866156+0000","last_fullsized":"2024-09-13T15:00:04.866156+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":533,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866146+0000","last_change":"2024-09-13T14:59:23.586799+0000","last_active":"2024-09-13T15:00:04.866146+0000","last_peered":"2024-09-13T15:00:04.866146+0000","last_clean":"2024-09-13T15:00:04.866146+0000","last_became_active":"2024-09-13T14:59:23.586428+0000","last_became_peered":"2024-09-13T14:59:23.586428+0000","last_unstale":"2024-09-13T15:00:04.866146+0000","last_undegraded":"2024-09-13T15:00:04.866146+0000","last_fullsized":"2024-09-13T15:00:04.866146+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:36.773654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":337,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865758+0000","last_change":"2024-09-13T14:58:38.887411+0000","last_active":"2024-09-13T15:00:04.865758+0000","last_peered":"2024-09-13T15:00:04.865758+0000","last_clean":"2024-09-13T15:00:04.865758+0000","last_became_active":"2024-09-13T14:58:37.388128+0000","last_became_peered":"2024-09-13T14:58:37.388128+0000","last_unstale":"2024-09-13T15:00:04.865758+0000","last_undegraded":"2024-09-13T15:00:04.865758+0000","last_fullsized":"2024-09-13T15:00:04.865758+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.887361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.887361+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:00.390537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":215,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864925+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T15:00:00.864925+0000","last_peered":"2024-09-13T15:00:00.864925+0000","last_clean":"2024-09-13T15:00:00.864925+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T15:00:00.864925+0000","last_undegraded":"2024-09-13T15:00:00.864925+0000","last_fullsized":"2024-09-13T15:00:00.864925+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":939,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866949+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T15:00:04.866949+0000","last_peered":"2024-09-13T15:00:04.866949+0000","last_clean":"2024-09-13T15:00:04.866949+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T15:00:04.866949+0000","last_undegraded":"2024-09-13T15:00:04.866949+0000","last_fullsized":"2024-09-13T15:00:04.866949+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":377,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682696+0000","last_change":"2024-09-13T14:58:45.170684+0000","last_active":"2024-09-13T15:00:00.682696+0000","last_peered":"2024-09-13T15:00:00.682696+0000","last_clean":"2024-09-13T15:00:00.682696+0000","last_became_active":"2024-09-13T14:58:37.388297+0000","last_became_peered":"2024-09-13T14:58:37.388297+0000","last_unstale":"2024-09-13T15:00:00.682696+0000","last_undegraded":"2024-09-13T15:00:00.682696+0000","last_fullsized":"2024-09-13T15:00:00.682696+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:45.170627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:45.170627+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:15.158824+0000","scrub_duration":0.0016372590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":890,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698159+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T15:00:03.698159+0000","last_peered":"2024-09-13T15:00:03.698159+0000","last_clean":"2024-09-13T15:00:03.698159+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T15:00:03.698159+0000","last_undegraded":"2024-09-13T15:00:03.698159+0000","last_fullsized":"2024-09-13T15:00:03.698159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866810+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T15:00:04.866810+0000","last_peered":"2024-09-13T15:00:04.866810+0000","last_clean":"2024-09-13T15:00:04.866810+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T15:00:04.866810+0000","last_undegraded":"2024-09-13T15:00:04.866810+0000","last_fullsized":"2024-09-13T15:00:04.866810+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":696,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682829+0000","last_change":"2024-09-13T14:58:49.080701+0000","last_active":"2024-09-13T15:00:00.682829+0000","last_peered":"2024-09-13T15:00:00.682829+0000","last_clean":"2024-09-13T15:00:00.682829+0000","last_became_active":"2024-09-13T14:58:37.388004+0000","last_became_peered":"2024-09-13T14:58:37.388004+0000","last_unstale":"2024-09-13T15:00:00.682829+0000","last_undegraded":"2024-09-13T15:00:00.682829+0000","last_fullsized":"2024-09-13T15:00:00.682829+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:49.080631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:49.080631+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:23:10.501463+0000","scrub_duration":0.00194838,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":958,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865310+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T15:00:04.865310+0000","last_peered":"2024-09-13T15:00:04.865310+0000","last_clean":"2024-09-13T15:00:04.865310+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T15:00:04.865310+0000","last_undegraded":"2024-09-13T15:00:04.865310+0000","last_fullsized":"2024-09-13T15:00:04.865310+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":515,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448371+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T15:00:01.448371+0000","last_peered":"2024-09-13T15:00:01.448371+0000","last_clean":"2024-09-13T15:00:01.448371+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T15:00:01.448371+0000","last_undegraded":"2024-09-13T15:00:01.448371+0000","last_fullsized":"2024-09-13T15:00:01.448371+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864683+0000","last_change":"2024-09-13T14:58:45.446295+0000","last_active":"2024-09-13T15:00:00.864683+0000","last_peered":"2024-09-13T15:00:00.864683+0000","last_clean":"2024-09-13T15:00:00.864683+0000","last_became_active":"2024-09-13T14:58:37.385240+0000","last_became_peered":"2024-09-13T14:58:37.385240+0000","last_unstale":"2024-09-13T15:00:00.864683+0000","last_undegraded":"2024-09-13T15:00:00.864683+0000","last_fullsized":"2024-09-13T15:00:00.864683+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:45.446237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:45.446237+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:38:14.187569+0000","scrub_duration":0.0013641339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865442+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T15:00:04.865442+0000","last_peered":"2024-09-13T15:00:04.865442+0000","last_clean":"2024-09-13T15:00:04.865442+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T15:00:04.865442+0000","last_undegraded":"2024-09-13T15:00:04.865442+0000","last_fullsized":"2024-09-13T15:00:04.865442+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866590+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T15:00:04.866590+0000","last_peered":"2024-09-13T15:00:04.866590+0000","last_clean":"2024-09-13T15:00:04.866590+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T15:00:04.866590+0000","last_undegraded":"2024-09-13T15:00:04.866590+0000","last_fullsized":"2024-09-13T15:00:04.866590+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759287+0000","last_change":"2024-09-13T14:58:46.493832+0000","last_active":"2024-09-13T15:00:00.759287+0000","last_peered":"2024-09-13T15:00:00.759287+0000","last_clean":"2024-09-13T15:00:00.759287+0000","last_became_active":"2024-09-13T14:58:37.388071+0000","last_became_peered":"2024-09-13T14:58:37.388071+0000","last_unstale":"2024-09-13T15:00:00.759287+0000","last_undegraded":"2024-09-13T15:00:00.759287+0000","last_fullsized":"2024-09-13T15:00:00.759287+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:46.493768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:46.493768+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:58.499664+0000","scrub_duration":0.001482084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":870,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865931+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T15:00:04.865931+0000","last_peered":"2024-09-13T15:00:04.865931+0000","last_clean":"2024-09-13T15:00:04.865931+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T15:00:04.865931+0000","last_undegraded":"2024-09-13T15:00:04.865931+0000","last_fullsized":"2024-09-13T15:00:04.865931+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683314+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T15:00:00.683314+0000","last_peered":"2024-09-13T15:00:00.683314+0000","last_clean":"2024-09-13T15:00:00.683314+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T15:00:00.683314+0000","last_undegraded":"2024-09-13T15:00:00.683314+0000","last_fullsized":"2024-09-13T15:00:00.683314+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":355,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866075+0000","last_change":"2024-09-13T14:58:08.732515+0000","last_active":"2024-09-13T15:00:04.866075+0000","last_peered":"2024-09-13T15:00:04.866075+0000","last_clean":"2024-09-13T15:00:04.866075+0000","last_became_active":"2024-09-13T14:58:07.825813+0000","last_became_peered":"2024-09-13T14:58:07.825813+0000","last_unstale":"2024-09-13T15:00:04.866075+0000","last_undegraded":"2024-09-13T15:00:04.866075+0000","last_fullsized":"2024-09-13T15:00:04.866075+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:08.732438+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.732438+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:07.684744+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":517,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759142+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T15:00:00.759142+0000","last_peered":"2024-09-13T15:00:00.759142+0000","last_clean":"2024-09-13T15:00:00.759142+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T15:00:00.759142+0000","last_undegraded":"2024-09-13T15:00:00.759142+0000","last_fullsized":"2024-09-13T15:00:00.759142+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":866,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698328+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T15:00:03.698328+0000","last_peered":"2024-09-13T15:00:03.698328+0000","last_clean":"2024-09-13T15:00:03.698328+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T15:00:03.698328+0000","last_undegraded":"2024-09-13T15:00:03.698328+0000","last_fullsized":"2024-09-13T15:00:03.698328+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683306+0000","last_change":"2024-09-13T14:59:01.058913+0000","last_active":"2024-09-13T15:00:00.683306+0000","last_peered":"2024-09-13T15:00:00.683306+0000","last_clean":"2024-09-13T15:00:00.683306+0000","last_became_active":"2024-09-13T14:58:37.387974+0000","last_became_peered":"2024-09-13T14:58:37.387974+0000","last_unstale":"2024-09-13T15:00:00.683306+0000","last_undegraded":"2024-09-13T15:00:00.683306+0000","last_fullsized":"2024-09-13T15:00:00.683306+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:59:01.058804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:59:01.058804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:24.486246+0000","scrub_duration":0.0030974800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":226,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864955+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T15:00:00.864955+0000","last_peered":"2024-09-13T15:00:00.864955+0000","last_clean":"2024-09-13T15:00:00.864955+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T15:00:00.864955+0000","last_undegraded":"2024-09-13T15:00:00.864955+0000","last_fullsized":"2024-09-13T15:00:00.864955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865092+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T15:00:00.865092+0000","last_peered":"2024-09-13T15:00:00.865092+0000","last_clean":"2024-09-13T15:00:00.865092+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T15:00:00.865092+0000","last_undegraded":"2024-09-13T15:00:00.865092+0000","last_fullsized":"2024-09-13T15:00:00.865092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":75,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698925+0000","last_change":"2024-09-13T14:58:38.696805+0000","last_active":"2024-09-13T15:00:03.698925+0000","last_peered":"2024-09-13T15:00:03.698925+0000","last_clean":"2024-09-13T15:00:03.698925+0000","last_became_active":"2024-09-13T14:58:37.386735+0000","last_became_peered":"2024-09-13T14:58:37.386735+0000","last_unstale":"2024-09-13T15:00:03.698925+0000","last_undegraded":"2024-09-13T15:00:03.698925+0000","last_fullsized":"2024-09-13T15:00:03.698925+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.696752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.696752+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:54.433469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":230,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864750+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T15:00:00.864750+0000","last_peered":"2024-09-13T15:00:00.864750+0000","last_clean":"2024-09-13T15:00:00.864750+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T15:00:00.864750+0000","last_undegraded":"2024-09-13T15:00:00.864750+0000","last_fullsized":"2024-09-13T15:00:00.864750+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":936,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448387+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T15:00:01.448387+0000","last_peered":"2024-09-13T15:00:01.448387+0000","last_clean":"2024-09-13T15:00:01.448387+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T15:00:01.448387+0000","last_undegraded":"2024-09-13T15:00:01.448387+0000","last_fullsized":"2024-09-13T15:00:01.448387+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":217,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865584+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T15:00:04.865584+0000","last_peered":"2024-09-13T15:00:04.865584+0000","last_clean":"2024-09-13T15:00:04.865584+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T15:00:04.865584+0000","last_undegraded":"2024-09-13T15:00:04.865584+0000","last_fullsized":"2024-09-13T15:00:04.865584+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":791,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866010+0000","last_change":"2024-09-13T14:56:00.000186+0000","last_active":"2024-09-13T15:00:04.866010+0000","last_peered":"2024-09-13T15:00:04.866010+0000","last_clean":"2024-09-13T15:00:04.866010+0000","last_became_active":"2024-09-13T14:55:54.611253+0000","last_became_peered":"2024-09-13T14:55:54.611253+0000","last_unstale":"2024-09-13T15:00:04.866010+0000","last_undegraded":"2024-09-13T15:00:04.866010+0000","last_fullsized":"2024-09-13T15:00:04.866010+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.999712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.999712+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:40.314935+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":927,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865530+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T15:00:04.865530+0000","last_peered":"2024-09-13T15:00:04.865530+0000","last_clean":"2024-09-13T15:00:04.865530+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T15:00:04.865530+0000","last_undegraded":"2024-09-13T15:00:04.865530+0000","last_fullsized":"2024-09-13T15:00:04.865530+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1255,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699116+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T15:00:03.699116+0000","last_peered":"2024-09-13T15:00:03.699116+0000","last_clean":"2024-09-13T15:00:03.699116+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T15:00:03.699116+0000","last_undegraded":"2024-09-13T15:00:03.699116+0000","last_fullsized":"2024-09-13T15:00:03.699116+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":937,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866397+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T15:00:04.866397+0000","last_peered":"2024-09-13T15:00:04.866397+0000","last_clean":"2024-09-13T15:00:04.866397+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T15:00:04.866397+0000","last_undegraded":"2024-09-13T15:00:04.866397+0000","last_fullsized":"2024-09-13T15:00:04.866397+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":256,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865211+0000","last_change":"2024-09-13T14:58:40.435743+0000","last_active":"2024-09-13T15:00:00.865211+0000","last_peered":"2024-09-13T15:00:00.865211+0000","last_clean":"2024-09-13T15:00:00.865211+0000","last_became_active":"2024-09-13T14:58:29.161014+0000","last_became_peered":"2024-09-13T14:58:29.161014+0000","last_unstale":"2024-09-13T15:00:00.865211+0000","last_undegraded":"2024-09-13T15:00:00.865211+0000","last_fullsized":"2024-09-13T15:00:00.865211+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:40.435644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:40.435644+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:03.521569+0000","scrub_duration":0.001852329,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865738+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T15:00:04.865738+0000","last_peered":"2024-09-13T15:00:04.865738+0000","last_clean":"2024-09-13T15:00:04.865738+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T15:00:04.865738+0000","last_undegraded":"2024-09-13T15:00:04.865738+0000","last_fullsized":"2024-09-13T15:00:04.865738+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":934,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759356+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T15:00:00.759356+0000","last_peered":"2024-09-13T15:00:00.759356+0000","last_clean":"2024-09-13T15:00:00.759356+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T15:00:00.759356+0000","last_undegraded":"2024-09-13T15:00:00.759356+0000","last_fullsized":"2024-09-13T15:00:00.759356+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":879,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698441+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T15:00:03.698441+0000","last_peered":"2024-09-13T15:00:03.698441+0000","last_clean":"2024-09-13T15:00:03.698441+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T15:00:03.698441+0000","last_undegraded":"2024-09-13T15:00:03.698441+0000","last_fullsized":"2024-09-13T15:00:03.698441+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":577,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448181+0000","last_change":"2024-09-13T14:58:40.905329+0000","last_active":"2024-09-13T15:00:01.448181+0000","last_peered":"2024-09-13T15:00:01.448181+0000","last_clean":"2024-09-13T15:00:01.448181+0000","last_became_active":"2024-09-13T14:58:29.161781+0000","last_became_peered":"2024-09-13T14:58:29.161781+0000","last_unstale":"2024-09-13T15:00:01.448181+0000","last_undegraded":"2024-09-13T15:00:01.448181+0000","last_fullsized":"2024-09-13T15:00:01.448181+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:40.905271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:40.905271+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:19.746868+0000","scrub_duration":0.001784119,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":870,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698392+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T15:00:03.698392+0000","last_peered":"2024-09-13T15:00:03.698392+0000","last_clean":"2024-09-13T15:00:03.698392+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T15:00:03.698392+0000","last_undegraded":"2024-09-13T15:00:03.698392+0000","last_fullsized":"2024-09-13T15:00:03.698392+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1552,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698546+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T15:00:03.698546+0000","last_peered":"2024-09-13T15:00:03.698546+0000","last_clean":"2024-09-13T15:00:03.698546+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T15:00:03.698546+0000","last_undegraded":"2024-09-13T15:00:03.698546+0000","last_fullsized":"2024-09-13T15:00:03.698546+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":463,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683312+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T15:00:00.683312+0000","last_peered":"2024-09-13T15:00:00.683312+0000","last_clean":"2024-09-13T15:00:00.683312+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T15:00:00.683312+0000","last_undegraded":"2024-09-13T15:00:00.683312+0000","last_fullsized":"2024-09-13T15:00:00.683312+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":557,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448303+0000","last_change":"2024-09-13T14:58:36.933856+0000","last_active":"2024-09-13T15:00:01.448303+0000","last_peered":"2024-09-13T15:00:01.448303+0000","last_clean":"2024-09-13T15:00:01.448303+0000","last_became_active":"2024-09-13T14:58:29.167655+0000","last_became_peered":"2024-09-13T14:58:29.167655+0000","last_unstale":"2024-09-13T15:00:01.448303+0000","last_undegraded":"2024-09-13T15:00:01.448303+0000","last_fullsized":"2024-09-13T15:00:01.448303+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.933789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.933789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:48:08.137668+0000","scrub_duration":0.0011416200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":889,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698393+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T15:00:03.698393+0000","last_peered":"2024-09-13T15:00:03.698393+0000","last_clean":"2024-09-13T15:00:03.698393+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T15:00:03.698393+0000","last_undegraded":"2024-09-13T15:00:03.698393+0000","last_fullsized":"2024-09-13T15:00:03.698393+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":236,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865071+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T15:00:00.865071+0000","last_peered":"2024-09-13T15:00:00.865071+0000","last_clean":"2024-09-13T15:00:00.865071+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T15:00:00.865071+0000","last_undegraded":"2024-09-13T15:00:00.865071+0000","last_fullsized":"2024-09-13T15:00:00.865071+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1133,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683108+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T15:00:00.683108+0000","last_peered":"2024-09-13T15:00:00.683108+0000","last_clean":"2024-09-13T15:00:00.683108+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T15:00:00.683108+0000","last_undegraded":"2024-09-13T15:00:00.683108+0000","last_fullsized":"2024-09-13T15:00:00.683108+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":376,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682846+0000","last_change":"2024-09-13T14:56:00.644560+0000","last_active":"2024-09-13T15:00:00.682846+0000","last_peered":"2024-09-13T15:00:00.682846+0000","last_clean":"2024-09-13T15:00:00.682846+0000","last_became_active":"2024-09-13T14:55:54.621234+0000","last_became_peered":"2024-09-13T14:55:54.621234+0000","last_unstale":"2024-09-13T15:00:00.682846+0000","last_undegraded":"2024-09-13T15:00:00.682846+0000","last_fullsized":"2024-09-13T15:00:00.682846+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.644466+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.644466+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.375763+0000","scrub_duration":0.0093267220000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":554,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866615+0000","last_change":"2024-09-13T14:58:07.832480+0000","last_active":"2024-09-13T15:00:04.866615+0000","last_peered":"2024-09-13T15:00:04.866615+0000","last_clean":"2024-09-13T15:00:04.866615+0000","last_became_active":"2024-09-13T14:58:07.831527+0000","last_became_peered":"2024-09-13T14:58:07.831527+0000","last_unstale":"2024-09-13T15:00:04.866615+0000","last_undegraded":"2024-09-13T15:00:04.866615+0000","last_fullsized":"2024-09-13T15:00:04.866615+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:25.525827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":869,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698228+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T15:00:03.698228+0000","last_peered":"2024-09-13T15:00:03.698228+0000","last_clean":"2024-09-13T15:00:03.698228+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T15:00:03.698228+0000","last_undegraded":"2024-09-13T15:00:03.698228+0000","last_fullsized":"2024-09-13T15:00:03.698228+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":928,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866999+0000","last_change":"2024-09-13T14:59:23.576669+0000","last_active":"2024-09-13T15:00:04.866999+0000","last_peered":"2024-09-13T15:00:04.866999+0000","last_clean":"2024-09-13T15:00:04.866999+0000","last_became_active":"2024-09-13T14:59:23.575951+0000","last_became_peered":"2024-09-13T14:59:23.575951+0000","last_unstale":"2024-09-13T15:00:04.866999+0000","last_undegraded":"2024-09-13T15:00:04.866999+0000","last_fullsized":"2024-09-13T15:00:04.866999+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:08:14.146814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":689,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758751+0000","last_change":"2024-09-13T14:55:58.908251+0000","last_active":"2024-09-13T15:00:00.758751+0000","last_peered":"2024-09-13T15:00:00.758751+0000","last_clean":"2024-09-13T15:00:00.758751+0000","last_became_active":"2024-09-13T14:55:54.619910+0000","last_became_peered":"2024-09-13T14:55:54.619910+0000","last_unstale":"2024-09-13T15:00:00.758751+0000","last_undegraded":"2024-09-13T15:00:00.758751+0000","last_fullsized":"2024-09-13T15:00:00.758751+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.908152+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.908152+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:01.217177+0000","scrub_duration":0.0048598549999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865727+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T15:00:04.865727+0000","last_peered":"2024-09-13T15:00:04.865727+0000","last_clean":"2024-09-13T15:00:04.865727+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T15:00:04.865727+0000","last_undegraded":"2024-09-13T15:00:04.865727+0000","last_fullsized":"2024-09-13T15:00:04.865727+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865541+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T15:00:04.865541+0000","last_peered":"2024-09-13T15:00:04.865541+0000","last_clean":"2024-09-13T15:00:04.865541+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T15:00:04.865541+0000","last_undegraded":"2024-09-13T15:00:04.865541+0000","last_fullsized":"2024-09-13T15:00:04.865541+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":930,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758828+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T15:00:00.758828+0000","last_peered":"2024-09-13T15:00:00.758828+0000","last_clean":"2024-09-13T15:00:00.758828+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T15:00:00.758828+0000","last_undegraded":"2024-09-13T15:00:00.758828+0000","last_fullsized":"2024-09-13T15:00:00.758828+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":761,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699204+0000","last_change":"2024-09-13T14:56:00.322316+0000","last_active":"2024-09-13T15:00:03.699204+0000","last_peered":"2024-09-13T15:00:03.699204+0000","last_clean":"2024-09-13T15:00:03.699204+0000","last_became_active":"2024-09-13T14:55:54.607589+0000","last_became_peered":"2024-09-13T14:55:54.607589+0000","last_unstale":"2024-09-13T15:00:03.699204+0000","last_undegraded":"2024-09-13T15:00:03.699204+0000","last_fullsized":"2024-09-13T15:00:03.699204+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.322230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.322230+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:58.559687+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":484,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448184+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T15:00:01.448184+0000","last_peered":"2024-09-13T15:00:01.448184+0000","last_clean":"2024-09-13T15:00:01.448184+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T15:00:01.448184+0000","last_undegraded":"2024-09-13T15:00:01.448184+0000","last_fullsized":"2024-09-13T15:00:01.448184+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":882,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865450+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T15:00:04.865450+0000","last_peered":"2024-09-13T15:00:04.865450+0000","last_clean":"2024-09-13T15:00:04.865450+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T15:00:04.865450+0000","last_undegraded":"2024-09-13T15:00:04.865450+0000","last_fullsized":"2024-09-13T15:00:04.865450+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":1042,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683039+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T15:00:00.683039+0000","last_peered":"2024-09-13T15:00:00.683039+0000","last_clean":"2024-09-13T15:00:00.683039+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T15:00:00.683039+0000","last_undegraded":"2024-09-13T15:00:00.683039+0000","last_fullsized":"2024-09-13T15:00:00.683039+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":660,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758921+0000","last_change":"2024-09-13T14:55:57.906683+0000","last_active":"2024-09-13T15:00:00.758921+0000","last_peered":"2024-09-13T15:00:00.758921+0000","last_clean":"2024-09-13T15:00:00.758921+0000","last_became_active":"2024-09-13T14:55:54.613915+0000","last_became_peered":"2024-09-13T14:55:54.613915+0000","last_unstale":"2024-09-13T15:00:00.758921+0000","last_undegraded":"2024-09-13T15:00:00.758921+0000","last_fullsized":"2024-09-13T15:00:00.758921+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.906630+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.906630+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:03.677753+0000","scrub_duration":0.001255616,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":890,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699326+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T15:00:03.699326+0000","last_peered":"2024-09-13T15:00:03.699326+0000","last_clean":"2024-09-13T15:00:03.699326+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T15:00:03.699326+0000","last_undegraded":"2024-09-13T15:00:03.699326+0000","last_fullsized":"2024-09-13T15:00:03.699326+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":928,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865636+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T15:00:04.865636+0000","last_peered":"2024-09-13T15:00:04.865636+0000","last_clean":"2024-09-13T15:00:04.865636+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T15:00:04.865636+0000","last_undegraded":"2024-09-13T15:00:04.865636+0000","last_fullsized":"2024-09-13T15:00:04.865636+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":356,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683039+0000","last_change":"2024-09-13T14:58:41.174258+0000","last_active":"2024-09-13T15:00:00.683039+0000","last_peered":"2024-09-13T15:00:00.683039+0000","last_clean":"2024-09-13T15:00:00.683039+0000","last_became_active":"2024-09-13T14:58:29.156586+0000","last_became_peered":"2024-09-13T14:58:29.156586+0000","last_unstale":"2024-09-13T15:00:00.683039+0000","last_undegraded":"2024-09-13T15:00:00.683039+0000","last_fullsized":"2024-09-13T15:00:00.683039+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:41.174201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:41.174201+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:22.118266+0000","scrub_duration":0.001866541,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865367+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T15:00:00.865367+0000","last_peered":"2024-09-13T15:00:00.865367+0000","last_clean":"2024-09-13T15:00:00.865367+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T15:00:00.865367+0000","last_undegraded":"2024-09-13T15:00:00.865367+0000","last_fullsized":"2024-09-13T15:00:00.865367+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864614+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T15:00:00.864614+0000","last_peered":"2024-09-13T15:00:00.864614+0000","last_clean":"2024-09-13T15:00:00.864614+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T15:00:00.864614+0000","last_undegraded":"2024-09-13T15:00:00.864614+0000","last_fullsized":"2024-09-13T15:00:00.864614+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":784,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699096+0000","last_change":"2024-09-13T14:58:36.631751+0000","last_active":"2024-09-13T15:00:03.699096+0000","last_peered":"2024-09-13T15:00:03.699096+0000","last_clean":"2024-09-13T15:00:03.699096+0000","last_became_active":"2024-09-13T14:58:29.163891+0000","last_became_peered":"2024-09-13T14:58:29.163891+0000","last_unstale":"2024-09-13T15:00:03.699096+0000","last_undegraded":"2024-09-13T15:00:03.699096+0000","last_fullsized":"2024-09-13T15:00:03.699096+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.631685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.631685+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:37.084679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865335+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T15:00:00.865335+0000","last_peered":"2024-09-13T15:00:00.865335+0000","last_clean":"2024-09-13T15:00:00.865335+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T15:00:00.865335+0000","last_undegraded":"2024-09-13T15:00:00.865335+0000","last_fullsized":"2024-09-13T15:00:00.865335+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.447855+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T15:00:01.447855+0000","last_peered":"2024-09-13T15:00:01.447855+0000","last_clean":"2024-09-13T15:00:01.447855+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T15:00:01.447855+0000","last_undegraded":"2024-09-13T15:00:01.447855+0000","last_fullsized":"2024-09-13T15:00:01.447855+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":363,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682964+0000","last_change":"2024-09-13T14:58:42.127362+0000","last_active":"2024-09-13T15:00:00.682964+0000","last_peered":"2024-09-13T15:00:00.682964+0000","last_clean":"2024-09-13T15:00:00.682964+0000","last_became_active":"2024-09-13T14:58:29.160077+0000","last_became_peered":"2024-09-13T14:58:29.160077+0000","last_unstale":"2024-09-13T15:00:00.682964+0000","last_undegraded":"2024-09-13T15:00:00.682964+0000","last_fullsized":"2024-09-13T15:00:00.682964+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:42.127256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:42.127256+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:54.811181+0000","scrub_duration":0.0018624659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865094+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T15:00:00.865094+0000","last_peered":"2024-09-13T15:00:00.865094+0000","last_clean":"2024-09-13T15:00:00.865094+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T15:00:00.865094+0000","last_undegraded":"2024-09-13T15:00:00.865094+0000","last_fullsized":"2024-09-13T15:00:00.865094+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":958,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865468+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T15:00:04.865468+0000","last_peered":"2024-09-13T15:00:04.865468+0000","last_clean":"2024-09-13T15:00:04.865468+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T15:00:04.865468+0000","last_undegraded":"2024-09-13T15:00:04.865468+0000","last_fullsized":"2024-09-13T15:00:04.865468+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":649,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758701+0000","last_change":"2024-09-13T14:58:38.479706+0000","last_active":"2024-09-13T15:00:00.758701+0000","last_peered":"2024-09-13T15:00:00.758701+0000","last_clean":"2024-09-13T15:00:00.758701+0000","last_became_active":"2024-09-13T14:58:29.163808+0000","last_became_peered":"2024-09-13T14:58:29.163808+0000","last_unstale":"2024-09-13T15:00:00.758701+0000","last_undegraded":"2024-09-13T15:00:00.758701+0000","last_fullsized":"2024-09-13T15:00:00.758701+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.479649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.479649+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:45.827480+0000","scrub_duration":0.0015113780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866266+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T15:00:04.866266+0000","last_peered":"2024-09-13T15:00:04.866266+0000","last_clean":"2024-09-13T15:00:04.866266+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T15:00:04.866266+0000","last_undegraded":"2024-09-13T15:00:04.866266+0000","last_fullsized":"2024-09-13T15:00:04.866266+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":446,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683117+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T15:00:00.683117+0000","last_peered":"2024-09-13T15:00:00.683117+0000","last_clean":"2024-09-13T15:00:00.683117+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T15:00:00.683117+0000","last_undegraded":"2024-09-13T15:00:00.683117+0000","last_fullsized":"2024-09-13T15:00:00.683117+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":91,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865194+0000","last_change":"2024-09-13T14:58:43.396350+0000","last_active":"2024-09-13T15:00:00.865194+0000","last_peered":"2024-09-13T15:00:00.865194+0000","last_clean":"2024-09-13T15:00:00.865194+0000","last_became_active":"2024-09-13T14:58:29.161667+0000","last_became_peered":"2024-09-13T14:58:29.161667+0000","last_unstale":"2024-09-13T15:00:00.865194+0000","last_undegraded":"2024-09-13T15:00:00.865194+0000","last_fullsized":"2024-09-13T15:00:00.865194+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:43.396294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:43.396294+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:08.269127+0000","scrub_duration":0.0013557140000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":498,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448162+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T15:00:01.448162+0000","last_peered":"2024-09-13T15:00:01.448162+0000","last_clean":"2024-09-13T15:00:01.448162+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T15:00:01.448162+0000","last_undegraded":"2024-09-13T15:00:01.448162+0000","last_fullsized":"2024-09-13T15:00:01.448162+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759272+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T15:00:00.759272+0000","last_peered":"2024-09-13T15:00:00.759272+0000","last_clean":"2024-09-13T15:00:00.759272+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T15:00:00.759272+0000","last_undegraded":"2024-09-13T15:00:00.759272+0000","last_fullsized":"2024-09-13T15:00:00.759272+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":546,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.447501+0000","last_change":"2024-09-13T14:56:02.698903+0000","last_active":"2024-09-13T15:00:01.447501+0000","last_peered":"2024-09-13T15:00:01.447501+0000","last_clean":"2024-09-13T15:00:01.447501+0000","last_became_active":"2024-09-13T14:56:00.691242+0000","last_became_peered":"2024-09-13T14:56:00.691242+0000","last_unstale":"2024-09-13T15:00:01.447501+0000","last_undegraded":"2024-09-13T15:00:01.447501+0000","last_fullsized":"2024-09-13T15:00:01.447501+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.698817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.698817+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:11.424244+0000","scrub_duration":0.0077410969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864851+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T15:00:00.864851+0000","last_peered":"2024-09-13T15:00:00.864851+0000","last_clean":"2024-09-13T15:00:00.864851+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T15:00:00.864851+0000","last_undegraded":"2024-09-13T15:00:00.864851+0000","last_fullsized":"2024-09-13T15:00:00.864851+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":529,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759318+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T15:00:00.759318+0000","last_peered":"2024-09-13T15:00:00.759318+0000","last_clean":"2024-09-13T15:00:00.759318+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T15:00:00.759318+0000","last_undegraded":"2024-09-13T15:00:00.759318+0000","last_fullsized":"2024-09-13T15:00:00.759318+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":821,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866163+0000","last_change":"2024-09-13T14:58:36.861034+0000","last_active":"2024-09-13T15:00:04.866163+0000","last_peered":"2024-09-13T15:00:04.866163+0000","last_clean":"2024-09-13T15:00:04.866163+0000","last_became_active":"2024-09-13T14:58:29.165260+0000","last_became_peered":"2024-09-13T14:58:29.165260+0000","last_unstale":"2024-09-13T15:00:04.866163+0000","last_undegraded":"2024-09-13T15:00:04.866163+0000","last_fullsized":"2024-09-13T15:00:04.866163+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.860981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.860981+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:35.870906+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":430,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682804+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T15:00:00.682804+0000","last_peered":"2024-09-13T15:00:00.682804+0000","last_clean":"2024-09-13T15:00:00.682804+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T15:00:00.682804+0000","last_undegraded":"2024-09-13T15:00:00.682804+0000","last_fullsized":"2024-09-13T15:00:00.682804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":539,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759353+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T15:00:00.759353+0000","last_peered":"2024-09-13T15:00:00.759353+0000","last_clean":"2024-09-13T15:00:00.759353+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T15:00:00.759353+0000","last_undegraded":"2024-09-13T15:00:00.759353+0000","last_fullsized":"2024-09-13T15:00:00.759353+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":91,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759070+0000","last_change":"2024-09-13T14:58:44.446102+0000","last_active":"2024-09-13T15:00:00.759070+0000","last_peered":"2024-09-13T15:00:00.759070+0000","last_clean":"2024-09-13T15:00:00.759070+0000","last_became_active":"2024-09-13T14:58:29.164312+0000","last_became_peered":"2024-09-13T14:58:29.164312+0000","last_unstale":"2024-09-13T15:00:00.759070+0000","last_undegraded":"2024-09-13T15:00:00.759070+0000","last_fullsized":"2024-09-13T15:00:00.759070+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:44.445993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:44.445993+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:34.621592+0000","scrub_duration":0.0014128000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":547,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448017+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T15:00:01.448017+0000","last_peered":"2024-09-13T15:00:01.448017+0000","last_clean":"2024-09-13T15:00:01.448017+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T15:00:01.448017+0000","last_undegraded":"2024-09-13T15:00:01.448017+0000","last_fullsized":"2024-09-13T15:00:01.448017+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866148+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T15:00:04.866148+0000","last_peered":"2024-09-13T15:00:04.866148+0000","last_clean":"2024-09-13T15:00:04.866148+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T15:00:04.866148+0000","last_undegraded":"2024-09-13T15:00:04.866148+0000","last_fullsized":"2024-09-13T15:00:04.866148+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":71,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683002+0000","last_change":"2024-09-13T14:56:01.635272+0000","last_active":"2024-09-13T15:00:00.683002+0000","last_peered":"2024-09-13T15:00:00.683002+0000","last_clean":"2024-09-13T15:00:00.683002+0000","last_became_active":"2024-09-13T14:56:00.667926+0000","last_became_peered":"2024-09-13T14:56:00.667926+0000","last_unstale":"2024-09-13T15:00:00.683002+0000","last_undegraded":"2024-09-13T15:00:00.683002+0000","last_fullsized":"2024-09-13T15:00:00.683002+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.635189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.635189+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:28.334884+0000","scrub_duration":0.010355512000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":485,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758973+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T15:00:00.758973+0000","last_peered":"2024-09-13T15:00:00.758973+0000","last_clean":"2024-09-13T15:00:00.758973+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T15:00:00.758973+0000","last_undegraded":"2024-09-13T15:00:00.758973+0000","last_fullsized":"2024-09-13T15:00:00.758973+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866762+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T15:00:04.866762+0000","last_peered":"2024-09-13T15:00:04.866762+0000","last_clean":"2024-09-13T15:00:04.866762+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T15:00:04.866762+0000","last_undegraded":"2024-09-13T15:00:04.866762+0000","last_fullsized":"2024-09-13T15:00:04.866762+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":767,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698942+0000","last_change":"2024-09-13T14:56:01.351509+0000","last_active":"2024-09-13T15:00:03.698942+0000","last_peered":"2024-09-13T15:00:03.698942+0000","last_clean":"2024-09-13T15:00:03.698942+0000","last_became_active":"2024-09-13T14:56:00.675249+0000","last_became_peered":"2024-09-13T14:56:00.675249+0000","last_unstale":"2024-09-13T15:00:03.698942+0000","last_undegraded":"2024-09-13T15:00:03.698942+0000","last_fullsized":"2024-09-13T15:00:03.698942+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.351298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.351298+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:36.329389+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866148+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T15:00:04.866148+0000","last_peered":"2024-09-13T15:00:04.866148+0000","last_clean":"2024-09-13T15:00:04.866148+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T15:00:04.866148+0000","last_undegraded":"2024-09-13T15:00:04.866148+0000","last_fullsized":"2024-09-13T15:00:04.866148+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":445,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683407+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T15:00:00.683407+0000","last_peered":"2024-09-13T15:00:00.683407+0000","last_clean":"2024-09-13T15:00:00.683407+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T15:00:00.683407+0000","last_undegraded":"2024-09-13T15:00:00.683407+0000","last_fullsized":"2024-09-13T15:00:00.683407+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":71,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682697+0000","last_change":"2024-09-13T14:56:04.574886+0000","last_active":"2024-09-13T15:00:00.682697+0000","last_peered":"2024-09-13T15:00:00.682697+0000","last_clean":"2024-09-13T15:00:00.682697+0000","last_became_active":"2024-09-13T14:56:00.674692+0000","last_became_peered":"2024-09-13T14:56:00.674692+0000","last_unstale":"2024-09-13T15:00:00.682697+0000","last_undegraded":"2024-09-13T15:00:00.682697+0000","last_fullsized":"2024-09-13T15:00:00.682697+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.574787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.574787+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:30.673194+0000","scrub_duration":0.0020770139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":516,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758956+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T15:00:00.758956+0000","last_peered":"2024-09-13T15:00:00.758956+0000","last_clean":"2024-09-13T15:00:00.758956+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T15:00:00.758956+0000","last_undegraded":"2024-09-13T15:00:00.758956+0000","last_fullsized":"2024-09-13T15:00:00.758956+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866273+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T15:00:04.866273+0000","last_peered":"2024-09-13T15:00:04.866273+0000","last_clean":"2024-09-13T15:00:04.866273+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T15:00:04.866273+0000","last_undegraded":"2024-09-13T15:00:04.866273+0000","last_fullsized":"2024-09-13T15:00:04.866273+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"82'100","reported_seq":377,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865752+0000","last_change":"2024-09-13T14:56:00.831620+0000","last_active":"2024-09-13T15:00:04.865752+0000","last_peered":"2024-09-13T15:00:04.865752+0000","last_clean":"2024-09-13T15:00:04.865752+0000","last_became_active":"2024-09-13T14:56:00.676063+0000","last_became_peered":"2024-09-13T14:56:00.676063+0000","last_unstale":"2024-09-13T15:00:04.865752+0000","last_undegraded":"2024-09-13T15:00:04.865752+0000","last_fullsized":"2024-09-13T15:00:04.865752+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.831575+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.831575+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:20.751784+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398098,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915340,"kb_used_data":11008,"kb_used_omap":1,"kb_used_meta":904318,"kb_avail":92829812,"statfs":{"total":95995035648,"available":95057727488,"internally_reserved":0,"allocated":11272192,"data_stored":8579930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926021976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594326,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905592,"kb_used_data":6956,"kb_used_omap":1,"kb_used_meta":898622,"kb_avail":92839560,"statfs":{"total":95995035648,"available":95067709440,"internally_reserved":0,"allocated":7122944,"data_stored":4449916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":920189272},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665779,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928756,"kb_used_data":6888,"kb_used_omap":9,"kb_used_meta":921846,"kb_avail":92816396,"statfs":{"total":95995035648,"available":95043989504,"internally_reserved":0,"allocated":7053312,"data_stored":4380687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":943970504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986782,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910660,"kb_used_data":6648,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":92834492,"statfs":{"total":95995035648,"available":95062519808,"internally_reserved":0,"allocated":6807552,"data_stored":4122868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697341,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146416,"kb_used_data":6948,"kb_used_omap":4,"kb_used_meta":1139451,"kb_avail":92598736,"statfs":{"total":95995035648,"available":94821105664,"internally_reserved":0,"allocated":7114752,"data_stored":4445760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1166797899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860868,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899768,"kb_used_data":6892,"kb_used_omap":8,"kb_used_meta":892855,"kb_avail":92845384,"statfs":{"total":95995035648,"available":95073673216,"internally_reserved":0,"allocated":7057408,"data_stored":4381282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":914284364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020539,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":680560,"kb_used_data":10724,"kb_used_omap":9,"kb_used_meta":669814,"kb_avail":93064592,"statfs":{"total":95995035648,"available":95298142208,"internally_reserved":0,"allocated":10981376,"data_stored":8312305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":685890373},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187930,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036464,"kb_used_data":11044,"kb_used_omap":7,"kb_used_meta":1025400,"kb_avail":92708688,"statfs":{"total":95995035648,"available":94933696512,"internally_reserved":0,"allocated":11309056,"data_stored":8640251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":1050009931},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:00:54.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:00:54.652 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:54 smithi189 ceph-mon[90992]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:55.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:54 smithi189 ceph-mon[90992]: from='client.47522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:55.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:54 smithi017 ceph-mon[103001]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:55.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:54 smithi017 ceph-mon[103001]: from='client.47522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:55.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:54 smithi017 ceph-mon[107556]: pgmap v1720: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:55.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:54 smithi017 ceph-mon[107556]: from='client.47522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:56.667 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-13T15:00:56.668 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-13T15:00:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:56 smithi017 ceph-mon[103001]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:56 smithi017 ceph-mon[107556]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:57.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:56 smithi189 ceph-mon[90992]: pgmap v1721: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-13T15:00:57.504 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1721,"stamp":"2024-09-13T15:00:55.528359+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":907,"num_write_kb":12623,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":8394258,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4132,"ondisk_log_size":4132,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7423556,"kb_used_data":67108,"kb_used_omap":44,"kb_used_meta":7356307,"kb_avail":742537660,"statfs":{"total":767960285184,"available":760358563840,"internally_reserved":0,"allocated":68718592,"data_stored":47312999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45833,"internal_metadata":7532858615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003181"},"pg_stats":[{"pgid":"3.1f","version":"64'4","reported_seq":219,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865436+0000","last_change":"2024-09-13T14:53:08.840594+0000","last_active":"2024-09-13T15:00:04.865436+0000","last_peered":"2024-09-13T15:00:04.865436+0000","last_clean":"2024-09-13T15:00:04.865436+0000","last_became_active":"2024-09-13T14:53:08.768608+0000","last_became_peered":"2024-09-13T14:53:08.768608+0000","last_unstale":"2024-09-13T15:00:04.865436+0000","last_undegraded":"2024-09-13T15:00:04.865436+0000","last_fullsized":"2024-09-13T15:00:04.865436+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:32.330370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864588+0000","last_change":"2024-09-13T14:40:26.253907+0000","last_active":"2024-09-13T15:00:00.864588+0000","last_peered":"2024-09-13T15:00:00.864588+0000","last_clean":"2024-09-13T15:00:00.864588+0000","last_became_active":"2024-09-13T14:40:26.253580+0000","last_became_peered":"2024-09-13T14:40:26.253580+0000","last_unstale":"2024-09-13T15:00:00.864588+0000","last_undegraded":"2024-09-13T15:00:00.864588+0000","last_fullsized":"2024-09-13T15:00:00.864588+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:13.319285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"82'100","reported_seq":77,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698373+0000","last_change":"2024-09-13T14:58:37.655579+0000","last_active":"2024-09-13T15:00:03.698373+0000","last_peered":"2024-09-13T15:00:03.698373+0000","last_clean":"2024-09-13T15:00:03.698373+0000","last_became_active":"2024-09-13T14:58:37.386557+0000","last_became_peered":"2024-09-13T14:58:37.386557+0000","last_unstale":"2024-09-13T15:00:03.698373+0000","last_undegraded":"2024-09-13T15:00:03.698373+0000","last_fullsized":"2024-09-13T15:00:03.698373+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:37.655535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:37.655535+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:15.463158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"64'1","reported_seq":924,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866767+0000","last_change":"2024-09-13T14:24:00.654003+0000","last_active":"2024-09-13T15:00:04.866767+0000","last_peered":"2024-09-13T15:00:04.866767+0000","last_clean":"2024-09-13T15:00:04.866767+0000","last_became_active":"2024-09-13T14:24:00.653625+0000","last_became_peered":"2024-09-13T14:24:00.653625+0000","last_unstale":"2024-09-13T15:00:04.866767+0000","last_undegraded":"2024-09-13T15:00:04.866767+0000","last_fullsized":"2024-09-13T15:00:04.866767+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:13.630116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":950,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866352+0000","last_change":"2024-09-13T14:53:08.800226+0000","last_active":"2024-09-13T15:00:04.866352+0000","last_peered":"2024-09-13T15:00:04.866352+0000","last_clean":"2024-09-13T15:00:04.866352+0000","last_became_active":"2024-09-13T14:53:08.785350+0000","last_became_peered":"2024-09-13T14:53:08.785350+0000","last_unstale":"2024-09-13T15:00:04.866352+0000","last_undegraded":"2024-09-13T15:00:04.866352+0000","last_fullsized":"2024-09-13T15:00:04.866352+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:37:46.222152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759021+0000","last_change":"2024-09-13T14:58:51.463423+0000","last_active":"2024-09-13T15:00:00.759021+0000","last_peered":"2024-09-13T15:00:00.759021+0000","last_clean":"2024-09-13T15:00:00.759021+0000","last_became_active":"2024-09-13T14:58:37.387887+0000","last_became_peered":"2024-09-13T14:58:37.387887+0000","last_unstale":"2024-09-13T15:00:00.759021+0000","last_undegraded":"2024-09-13T15:00:00.759021+0000","last_fullsized":"2024-09-13T15:00:00.759021+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:51.463355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:51.463355+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:45.169469+0000","scrub_duration":0.0018833960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":449,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683402+0000","last_change":"2024-09-13T14:40:45.721597+0000","last_active":"2024-09-13T15:00:00.683402+0000","last_peered":"2024-09-13T15:00:00.683402+0000","last_clean":"2024-09-13T15:00:00.683402+0000","last_became_active":"2024-09-13T14:40:45.721248+0000","last_became_peered":"2024-09-13T14:40:45.721248+0000","last_unstale":"2024-09-13T15:00:00.683402+0000","last_undegraded":"2024-09-13T15:00:00.683402+0000","last_fullsized":"2024-09-13T15:00:00.683402+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:51.790160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":235,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864865+0000","last_change":"2024-09-13T14:53:08.774299+0000","last_active":"2024-09-13T15:00:00.864865+0000","last_peered":"2024-09-13T15:00:00.864865+0000","last_clean":"2024-09-13T15:00:00.864865+0000","last_became_active":"2024-09-13T14:53:08.774030+0000","last_became_peered":"2024-09-13T14:53:08.774030+0000","last_unstale":"2024-09-13T15:00:00.864865+0000","last_undegraded":"2024-09-13T15:00:00.864865+0000","last_fullsized":"2024-09-13T15:00:00.864865+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:35.208682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683145+0000","last_change":"2024-09-13T14:40:45.697483+0000","last_active":"2024-09-13T15:00:00.683145+0000","last_peered":"2024-09-13T15:00:00.683145+0000","last_clean":"2024-09-13T15:00:00.683145+0000","last_became_active":"2024-09-13T14:40:45.697155+0000","last_became_peered":"2024-09-13T14:40:45.697155+0000","last_unstale":"2024-09-13T15:00:00.683145+0000","last_undegraded":"2024-09-13T15:00:00.683145+0000","last_fullsized":"2024-09-13T15:00:00.683145+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:15.712114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864973+0000","last_change":"2024-09-13T14:40:26.265426+0000","last_active":"2024-09-13T15:00:00.864973+0000","last_peered":"2024-09-13T15:00:00.864973+0000","last_clean":"2024-09-13T15:00:00.864973+0000","last_became_active":"2024-09-13T14:40:26.265106+0000","last_became_peered":"2024-09-13T14:40:26.265106+0000","last_unstale":"2024-09-13T15:00:00.864973+0000","last_undegraded":"2024-09-13T15:00:00.864973+0000","last_fullsized":"2024-09-13T15:00:00.864973+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:37.932843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"82'100","reported_seq":226,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866761+0000","last_change":"2024-09-13T14:58:37.837560+0000","last_active":"2024-09-13T15:00:04.866761+0000","last_peered":"2024-09-13T15:00:04.866761+0000","last_clean":"2024-09-13T15:00:04.866761+0000","last_became_active":"2024-09-13T14:58:37.385007+0000","last_became_peered":"2024-09-13T14:58:37.385007+0000","last_unstale":"2024-09-13T15:00:04.866761+0000","last_undegraded":"2024-09-13T15:00:04.866761+0000","last_fullsized":"2024-09-13T15:00:04.866761+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:37.837514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:37.837514+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:33:39.369516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"64'1","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866156+0000","last_change":"2024-09-13T14:53:08.769219+0000","last_active":"2024-09-13T15:00:04.866156+0000","last_peered":"2024-09-13T15:00:04.866156+0000","last_clean":"2024-09-13T15:00:04.866156+0000","last_became_active":"2024-09-13T14:53:08.768725+0000","last_became_peered":"2024-09-13T14:53:08.768725+0000","last_unstale":"2024-09-13T15:00:04.866156+0000","last_undegraded":"2024-09-13T15:00:04.866156+0000","last_fullsized":"2024-09-13T15:00:04.866156+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:28.198080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":533,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866146+0000","last_change":"2024-09-13T14:59:23.586799+0000","last_active":"2024-09-13T15:00:04.866146+0000","last_peered":"2024-09-13T15:00:04.866146+0000","last_clean":"2024-09-13T15:00:04.866146+0000","last_became_active":"2024-09-13T14:59:23.586428+0000","last_became_peered":"2024-09-13T14:59:23.586428+0000","last_unstale":"2024-09-13T15:00:04.866146+0000","last_undegraded":"2024-09-13T15:00:04.866146+0000","last_fullsized":"2024-09-13T15:00:04.866146+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:36.773654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"82'100","reported_seq":337,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865758+0000","last_change":"2024-09-13T14:58:38.887411+0000","last_active":"2024-09-13T15:00:04.865758+0000","last_peered":"2024-09-13T15:00:04.865758+0000","last_clean":"2024-09-13T15:00:04.865758+0000","last_became_active":"2024-09-13T14:58:37.388128+0000","last_became_peered":"2024-09-13T14:58:37.388128+0000","last_unstale":"2024-09-13T15:00:04.865758+0000","last_undegraded":"2024-09-13T15:00:04.865758+0000","last_fullsized":"2024-09-13T15:00:04.865758+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.887361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.887361+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:00.390537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"64'1","reported_seq":215,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864925+0000","last_change":"2024-09-13T14:40:26.263419+0000","last_active":"2024-09-13T15:00:00.864925+0000","last_peered":"2024-09-13T15:00:00.864925+0000","last_clean":"2024-09-13T15:00:00.864925+0000","last_became_active":"2024-09-13T14:40:26.263050+0000","last_became_peered":"2024-09-13T14:40:26.263050+0000","last_unstale":"2024-09-13T15:00:00.864925+0000","last_undegraded":"2024-09-13T15:00:00.864925+0000","last_fullsized":"2024-09-13T15:00:00.864925+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:47.095976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":939,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866949+0000","last_change":"2024-09-13T14:40:26.265521+0000","last_active":"2024-09-13T15:00:04.866949+0000","last_peered":"2024-09-13T15:00:04.866949+0000","last_clean":"2024-09-13T15:00:04.866949+0000","last_became_active":"2024-09-13T14:40:26.265092+0000","last_became_peered":"2024-09-13T14:40:26.265092+0000","last_unstale":"2024-09-13T15:00:04.866949+0000","last_undegraded":"2024-09-13T15:00:04.866949+0000","last_fullsized":"2024-09-13T15:00:04.866949+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:48.652317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"82'100","reported_seq":377,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682696+0000","last_change":"2024-09-13T14:58:45.170684+0000","last_active":"2024-09-13T15:00:00.682696+0000","last_peered":"2024-09-13T15:00:00.682696+0000","last_clean":"2024-09-13T15:00:00.682696+0000","last_became_active":"2024-09-13T14:58:37.388297+0000","last_became_peered":"2024-09-13T14:58:37.388297+0000","last_unstale":"2024-09-13T15:00:00.682696+0000","last_undegraded":"2024-09-13T15:00:00.682696+0000","last_fullsized":"2024-09-13T15:00:00.682696+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:45.170627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:45.170627+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:15.158824+0000","scrub_duration":0.0016372590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":890,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698159+0000","last_change":"2024-09-13T14:40:26.268407+0000","last_active":"2024-09-13T15:00:03.698159+0000","last_peered":"2024-09-13T15:00:03.698159+0000","last_clean":"2024-09-13T15:00:03.698159+0000","last_became_active":"2024-09-13T14:40:26.268031+0000","last_became_peered":"2024-09-13T14:40:26.268031+0000","last_unstale":"2024-09-13T15:00:03.698159+0000","last_undegraded":"2024-09-13T15:00:03.698159+0000","last_fullsized":"2024-09-13T15:00:03.698159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:56.826856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866810+0000","last_change":"2024-09-13T14:53:08.795371+0000","last_active":"2024-09-13T15:00:04.866810+0000","last_peered":"2024-09-13T15:00:04.866810+0000","last_clean":"2024-09-13T15:00:04.866810+0000","last_became_active":"2024-09-13T14:53:08.794916+0000","last_became_peered":"2024-09-13T14:53:08.794916+0000","last_unstale":"2024-09-13T15:00:04.866810+0000","last_undegraded":"2024-09-13T15:00:04.866810+0000","last_fullsized":"2024-09-13T15:00:04.866810+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:17.792115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"82'100","reported_seq":696,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682829+0000","last_change":"2024-09-13T14:58:49.080701+0000","last_active":"2024-09-13T15:00:00.682829+0000","last_peered":"2024-09-13T15:00:00.682829+0000","last_clean":"2024-09-13T15:00:00.682829+0000","last_became_active":"2024-09-13T14:58:37.388004+0000","last_became_peered":"2024-09-13T14:58:37.388004+0000","last_unstale":"2024-09-13T15:00:00.682829+0000","last_undegraded":"2024-09-13T15:00:00.682829+0000","last_fullsized":"2024-09-13T15:00:00.682829+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:49.080631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:49.080631+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:23:10.501463+0000","scrub_duration":0.00194838,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":958,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865310+0000","last_change":"2024-09-13T14:53:08.843332+0000","last_active":"2024-09-13T15:00:04.865310+0000","last_peered":"2024-09-13T15:00:04.865310+0000","last_clean":"2024-09-13T15:00:04.865310+0000","last_became_active":"2024-09-13T14:53:08.825725+0000","last_became_peered":"2024-09-13T14:53:08.825725+0000","last_unstale":"2024-09-13T15:00:04.865310+0000","last_undegraded":"2024-09-13T15:00:04.865310+0000","last_fullsized":"2024-09-13T15:00:04.865310+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:52.612310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":515,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448371+0000","last_change":"2024-09-13T14:40:44.708178+0000","last_active":"2024-09-13T15:00:01.448371+0000","last_peered":"2024-09-13T15:00:01.448371+0000","last_clean":"2024-09-13T15:00:01.448371+0000","last_became_active":"2024-09-13T14:40:44.707882+0000","last_became_peered":"2024-09-13T14:40:44.707882+0000","last_unstale":"2024-09-13T15:00:01.448371+0000","last_undegraded":"2024-09-13T15:00:01.448371+0000","last_fullsized":"2024-09-13T15:00:01.448371+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:31:08.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864683+0000","last_change":"2024-09-13T14:58:45.446295+0000","last_active":"2024-09-13T15:00:00.864683+0000","last_peered":"2024-09-13T15:00:00.864683+0000","last_clean":"2024-09-13T15:00:00.864683+0000","last_became_active":"2024-09-13T14:58:37.385240+0000","last_became_peered":"2024-09-13T14:58:37.385240+0000","last_unstale":"2024-09-13T15:00:00.864683+0000","last_undegraded":"2024-09-13T15:00:00.864683+0000","last_fullsized":"2024-09-13T15:00:00.864683+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:45.446237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:45.446237+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:38:14.187569+0000","scrub_duration":0.0013641339999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865442+0000","last_change":"2024-09-13T14:34:46.565965+0000","last_active":"2024-09-13T15:00:04.865442+0000","last_peered":"2024-09-13T15:00:04.865442+0000","last_clean":"2024-09-13T15:00:04.865442+0000","last_became_active":"2024-09-13T14:34:46.564707+0000","last_became_peered":"2024-09-13T14:34:46.564707+0000","last_unstale":"2024-09-13T15:00:04.865442+0000","last_undegraded":"2024-09-13T15:00:04.865442+0000","last_fullsized":"2024-09-13T15:00:04.865442+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:14.140024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866590+0000","last_change":"2024-09-13T14:53:08.807268+0000","last_active":"2024-09-13T15:00:04.866590+0000","last_peered":"2024-09-13T15:00:04.866590+0000","last_clean":"2024-09-13T15:00:04.866590+0000","last_became_active":"2024-09-13T14:53:08.806731+0000","last_became_peered":"2024-09-13T14:53:08.806731+0000","last_unstale":"2024-09-13T15:00:04.866590+0000","last_undegraded":"2024-09-13T15:00:04.866590+0000","last_fullsized":"2024-09-13T15:00:04.866590+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:31:02.199932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759287+0000","last_change":"2024-09-13T14:58:46.493832+0000","last_active":"2024-09-13T15:00:00.759287+0000","last_peered":"2024-09-13T15:00:00.759287+0000","last_clean":"2024-09-13T15:00:00.759287+0000","last_became_active":"2024-09-13T14:58:37.388071+0000","last_became_peered":"2024-09-13T14:58:37.388071+0000","last_unstale":"2024-09-13T15:00:00.759287+0000","last_undegraded":"2024-09-13T15:00:00.759287+0000","last_fullsized":"2024-09-13T15:00:00.759287+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:46.493768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:46.493768+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:58.499664+0000","scrub_duration":0.001482084,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":870,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865931+0000","last_change":"2024-09-13T14:24:00.657729+0000","last_active":"2024-09-13T15:00:04.865931+0000","last_peered":"2024-09-13T15:00:04.865931+0000","last_clean":"2024-09-13T15:00:04.865931+0000","last_became_active":"2024-09-13T14:24:00.656071+0000","last_became_peered":"2024-09-13T14:24:00.656071+0000","last_unstale":"2024-09-13T15:00:04.865931+0000","last_undegraded":"2024-09-13T15:00:04.865931+0000","last_fullsized":"2024-09-13T15:00:04.865931+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:25.994840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683314+0000","last_change":"2024-09-13T14:40:44.703232+0000","last_active":"2024-09-13T15:00:00.683314+0000","last_peered":"2024-09-13T15:00:00.683314+0000","last_clean":"2024-09-13T15:00:00.683314+0000","last_became_active":"2024-09-13T14:40:44.702956+0000","last_became_peered":"2024-09-13T14:40:44.702956+0000","last_unstale":"2024-09-13T15:00:00.683314+0000","last_undegraded":"2024-09-13T15:00:00.683314+0000","last_fullsized":"2024-09-13T15:00:00.683314+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:58.893120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"82'100","reported_seq":355,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866075+0000","last_change":"2024-09-13T14:58:08.732515+0000","last_active":"2024-09-13T15:00:04.866075+0000","last_peered":"2024-09-13T15:00:04.866075+0000","last_clean":"2024-09-13T15:00:04.866075+0000","last_became_active":"2024-09-13T14:58:07.825813+0000","last_became_peered":"2024-09-13T14:58:07.825813+0000","last_unstale":"2024-09-13T15:00:04.866075+0000","last_undegraded":"2024-09-13T15:00:04.866075+0000","last_fullsized":"2024-09-13T15:00:04.866075+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:08.732438+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.732438+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:07.684744+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":517,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759142+0000","last_change":"2024-09-13T14:40:26.270328+0000","last_active":"2024-09-13T15:00:00.759142+0000","last_peered":"2024-09-13T15:00:00.759142+0000","last_clean":"2024-09-13T15:00:00.759142+0000","last_became_active":"2024-09-13T14:40:26.257169+0000","last_became_peered":"2024-09-13T14:40:26.257169+0000","last_unstale":"2024-09-13T15:00:00.759142+0000","last_undegraded":"2024-09-13T15:00:00.759142+0000","last_fullsized":"2024-09-13T15:00:00.759142+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:34.250093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":866,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698328+0000","last_change":"2024-09-13T14:34:46.617844+0000","last_active":"2024-09-13T15:00:03.698328+0000","last_peered":"2024-09-13T15:00:03.698328+0000","last_clean":"2024-09-13T15:00:03.698328+0000","last_became_active":"2024-09-13T14:34:46.617436+0000","last_became_peered":"2024-09-13T14:34:46.617436+0000","last_unstale":"2024-09-13T15:00:03.698328+0000","last_undegraded":"2024-09-13T15:00:03.698328+0000","last_fullsized":"2024-09-13T15:00:03.698328+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:28.868813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"82'100","reported_seq":47,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683306+0000","last_change":"2024-09-13T14:59:01.058913+0000","last_active":"2024-09-13T15:00:00.683306+0000","last_peered":"2024-09-13T15:00:00.683306+0000","last_clean":"2024-09-13T15:00:00.683306+0000","last_became_active":"2024-09-13T14:58:37.387974+0000","last_became_peered":"2024-09-13T14:58:37.387974+0000","last_unstale":"2024-09-13T15:00:00.683306+0000","last_undegraded":"2024-09-13T15:00:00.683306+0000","last_fullsized":"2024-09-13T15:00:00.683306+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:59:01.058804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:59:01.058804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:24.486246+0000","scrub_duration":0.0030974800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":226,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864955+0000","last_change":"2024-09-13T14:40:26.268687+0000","last_active":"2024-09-13T15:00:00.864955+0000","last_peered":"2024-09-13T15:00:00.864955+0000","last_clean":"2024-09-13T15:00:00.864955+0000","last_became_active":"2024-09-13T14:40:26.255517+0000","last_became_peered":"2024-09-13T14:40:26.255517+0000","last_unstale":"2024-09-13T15:00:00.864955+0000","last_undegraded":"2024-09-13T15:00:00.864955+0000","last_fullsized":"2024-09-13T15:00:00.864955+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:39.485878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865092+0000","last_change":"2024-09-13T14:40:26.265307+0000","last_active":"2024-09-13T15:00:00.865092+0000","last_peered":"2024-09-13T15:00:00.865092+0000","last_clean":"2024-09-13T15:00:00.865092+0000","last_became_active":"2024-09-13T14:40:26.264805+0000","last_became_peered":"2024-09-13T14:40:26.264805+0000","last_unstale":"2024-09-13T15:00:00.865092+0000","last_undegraded":"2024-09-13T15:00:00.865092+0000","last_fullsized":"2024-09-13T15:00:00.865092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:15.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"82'100","reported_seq":75,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698925+0000","last_change":"2024-09-13T14:58:38.696805+0000","last_active":"2024-09-13T15:00:03.698925+0000","last_peered":"2024-09-13T15:00:03.698925+0000","last_clean":"2024-09-13T15:00:03.698925+0000","last_became_active":"2024-09-13T14:58:37.386735+0000","last_became_peered":"2024-09-13T14:58:37.386735+0000","last_unstale":"2024-09-13T15:00:03.698925+0000","last_undegraded":"2024-09-13T15:00:03.698925+0000","last_fullsized":"2024-09-13T15:00:03.698925+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":487,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.696752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.696752+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:54.433469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":230,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864750+0000","last_change":"2024-09-13T14:40:45.697952+0000","last_active":"2024-09-13T15:00:00.864750+0000","last_peered":"2024-09-13T15:00:00.864750+0000","last_clean":"2024-09-13T15:00:00.864750+0000","last_became_active":"2024-09-13T14:40:45.697679+0000","last_became_peered":"2024-09-13T14:40:45.697679+0000","last_unstale":"2024-09-13T15:00:00.864750+0000","last_undegraded":"2024-09-13T15:00:00.864750+0000","last_fullsized":"2024-09-13T15:00:00.864750+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:20.891608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'92","reported_seq":936,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448387+0000","last_change":"2024-09-13T14:23:27.721087+0000","last_active":"2024-09-13T15:00:01.448387+0000","last_peered":"2024-09-13T15:00:01.448387+0000","last_clean":"2024-09-13T15:00:01.448387+0000","last_became_active":"2024-09-13T14:23:27.720666+0000","last_became_peered":"2024-09-13T14:23:27.720666+0000","last_unstale":"2024-09-13T15:00:01.448387+0000","last_undegraded":"2024-09-13T15:00:01.448387+0000","last_fullsized":"2024-09-13T15:00:01.448387+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:49.011266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091195700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":848,"num_read_kb":773,"num_write":74,"num_write_kb":1070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"64'1","reported_seq":217,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865584+0000","last_change":"2024-09-13T14:53:08.870399+0000","last_active":"2024-09-13T15:00:04.865584+0000","last_peered":"2024-09-13T15:00:04.865584+0000","last_clean":"2024-09-13T15:00:04.865584+0000","last_became_active":"2024-09-13T14:53:08.772711+0000","last_became_peered":"2024-09-13T14:53:08.772711+0000","last_unstale":"2024-09-13T15:00:04.865584+0000","last_undegraded":"2024-09-13T15:00:04.865584+0000","last_fullsized":"2024-09-13T15:00:04.865584+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:20.050509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"82'100","reported_seq":791,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866010+0000","last_change":"2024-09-13T14:56:00.000186+0000","last_active":"2024-09-13T15:00:04.866010+0000","last_peered":"2024-09-13T15:00:04.866010+0000","last_clean":"2024-09-13T15:00:04.866010+0000","last_became_active":"2024-09-13T14:55:54.611253+0000","last_became_peered":"2024-09-13T14:55:54.611253+0000","last_unstale":"2024-09-13T15:00:04.866010+0000","last_undegraded":"2024-09-13T15:00:04.866010+0000","last_fullsized":"2024-09-13T15:00:04.866010+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:59.999712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:59.999712+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:40.314935+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":927,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865530+0000","last_change":"2024-09-13T14:53:08.789408+0000","last_active":"2024-09-13T15:00:04.865530+0000","last_peered":"2024-09-13T15:00:04.865530+0000","last_clean":"2024-09-13T15:00:04.865530+0000","last_became_active":"2024-09-13T14:53:08.788554+0000","last_became_peered":"2024-09-13T14:53:08.788554+0000","last_unstale":"2024-09-13T15:00:04.865530+0000","last_undegraded":"2024-09-13T15:00:04.865530+0000","last_fullsized":"2024-09-13T15:00:04.865530+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:34.719617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'79","reported_seq":1255,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699116+0000","last_change":"2024-09-13T14:40:26.282174+0000","last_active":"2024-09-13T15:00:03.699116+0000","last_peered":"2024-09-13T15:00:03.699116+0000","last_clean":"2024-09-13T15:00:03.699116+0000","last_became_active":"2024-09-13T14:40:26.266288+0000","last_became_peered":"2024-09-13T14:40:26.266288+0000","last_unstale":"2024-09-13T15:00:03.699116+0000","last_undegraded":"2024-09-13T15:00:03.699116+0000","last_fullsized":"2024-09-13T15:00:03.699116+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:35.157924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038846099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":509,"num_read_kb":708,"num_write":39,"num_write_kb":174,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":937,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866397+0000","last_change":"2024-09-13T14:53:09.040005+0000","last_active":"2024-09-13T15:00:04.866397+0000","last_peered":"2024-09-13T15:00:04.866397+0000","last_clean":"2024-09-13T15:00:04.866397+0000","last_became_active":"2024-09-13T14:53:08.874585+0000","last_became_peered":"2024-09-13T14:53:08.874585+0000","last_unstale":"2024-09-13T15:00:04.866397+0000","last_undegraded":"2024-09-13T15:00:04.866397+0000","last_fullsized":"2024-09-13T15:00:04.866397+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:00.605497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"82'100","reported_seq":256,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865211+0000","last_change":"2024-09-13T14:58:40.435743+0000","last_active":"2024-09-13T15:00:00.865211+0000","last_peered":"2024-09-13T15:00:00.865211+0000","last_clean":"2024-09-13T15:00:00.865211+0000","last_became_active":"2024-09-13T14:58:29.161014+0000","last_became_peered":"2024-09-13T14:58:29.161014+0000","last_unstale":"2024-09-13T15:00:00.865211+0000","last_undegraded":"2024-09-13T15:00:00.865211+0000","last_fullsized":"2024-09-13T15:00:00.865211+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:40.435644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:40.435644+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:03.521569+0000","scrub_duration":0.001852329,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865738+0000","last_change":"2024-09-13T14:53:08.768740+0000","last_active":"2024-09-13T15:00:04.865738+0000","last_peered":"2024-09-13T15:00:04.865738+0000","last_clean":"2024-09-13T15:00:04.865738+0000","last_became_active":"2024-09-13T14:53:08.768330+0000","last_became_peered":"2024-09-13T14:53:08.768330+0000","last_unstale":"2024-09-13T15:00:04.865738+0000","last_undegraded":"2024-09-13T15:00:04.865738+0000","last_fullsized":"2024-09-13T15:00:04.865738+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:12.458233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'112","reported_seq":934,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759356+0000","last_change":"2024-09-13T14:53:09.824898+0000","last_active":"2024-09-13T15:00:00.759356+0000","last_peered":"2024-09-13T15:00:00.759356+0000","last_clean":"2024-09-13T15:00:00.759356+0000","last_became_active":"2024-09-13T14:53:09.824385+0000","last_became_peered":"2024-09-13T14:53:09.824385+0000","last_unstale":"2024-09-13T15:00:00.759356+0000","last_undegraded":"2024-09-13T15:00:00.759356+0000","last_fullsized":"2024-09-13T15:00:00.759356+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:12.846329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034008399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":577,"num_read_kb":2509,"num_write":109,"num_write_kb":2095,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":879,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698441+0000","last_change":"2024-09-13T14:40:44.709705+0000","last_active":"2024-09-13T15:00:03.698441+0000","last_peered":"2024-09-13T15:00:03.698441+0000","last_clean":"2024-09-13T15:00:03.698441+0000","last_became_active":"2024-09-13T14:40:44.709325+0000","last_became_peered":"2024-09-13T14:40:44.709325+0000","last_unstale":"2024-09-13T15:00:03.698441+0000","last_undegraded":"2024-09-13T15:00:03.698441+0000","last_fullsized":"2024-09-13T15:00:03.698441+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:08:40.607072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'100","reported_seq":577,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448181+0000","last_change":"2024-09-13T14:58:40.905329+0000","last_active":"2024-09-13T15:00:01.448181+0000","last_peered":"2024-09-13T15:00:01.448181+0000","last_clean":"2024-09-13T15:00:01.448181+0000","last_became_active":"2024-09-13T14:58:29.161781+0000","last_became_peered":"2024-09-13T14:58:29.161781+0000","last_unstale":"2024-09-13T15:00:01.448181+0000","last_undegraded":"2024-09-13T15:00:01.448181+0000","last_fullsized":"2024-09-13T15:00:01.448181+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:40.905271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:40.905271+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:19.746868+0000","scrub_duration":0.001784119,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":870,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698392+0000","last_change":"2024-09-13T14:41:02.108647+0000","last_active":"2024-09-13T15:00:03.698392+0000","last_peered":"2024-09-13T15:00:03.698392+0000","last_clean":"2024-09-13T15:00:03.698392+0000","last_became_active":"2024-09-13T14:41:02.108363+0000","last_became_peered":"2024-09-13T14:41:02.108363+0000","last_unstale":"2024-09-13T15:00:03.698392+0000","last_undegraded":"2024-09-13T15:00:03.698392+0000","last_fullsized":"2024-09-13T15:00:03.698392+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:37.828861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'183","reported_seq":1552,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698546+0000","last_change":"2024-09-13T14:54:02.645915+0000","last_active":"2024-09-13T15:00:03.698546+0000","last_peered":"2024-09-13T15:00:03.698546+0000","last_clean":"2024-09-13T15:00:03.698546+0000","last_became_active":"2024-09-13T14:54:02.645168+0000","last_became_peered":"2024-09-13T14:54:02.645168+0000","last_unstale":"2024-09-13T15:00:03.698546+0000","last_undegraded":"2024-09-13T15:00:03.698546+0000","last_fullsized":"2024-09-13T15:00:03.698546+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:39.767806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054542499999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":882,"num_read_kb":2182,"num_write":183,"num_write_kb":5400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":463,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683312+0000","last_change":"2024-09-13T14:40:44.705226+0000","last_active":"2024-09-13T15:00:00.683312+0000","last_peered":"2024-09-13T15:00:00.683312+0000","last_clean":"2024-09-13T15:00:00.683312+0000","last_became_active":"2024-09-13T14:40:44.704562+0000","last_became_peered":"2024-09-13T14:40:44.704562+0000","last_unstale":"2024-09-13T15:00:00.683312+0000","last_undegraded":"2024-09-13T15:00:00.683312+0000","last_fullsized":"2024-09-13T15:00:00.683312+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:53.629701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"82'100","reported_seq":557,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448303+0000","last_change":"2024-09-13T14:58:36.933856+0000","last_active":"2024-09-13T15:00:01.448303+0000","last_peered":"2024-09-13T15:00:01.448303+0000","last_clean":"2024-09-13T15:00:01.448303+0000","last_became_active":"2024-09-13T14:58:29.167655+0000","last_became_peered":"2024-09-13T14:58:29.167655+0000","last_unstale":"2024-09-13T15:00:01.448303+0000","last_undegraded":"2024-09-13T15:00:01.448303+0000","last_fullsized":"2024-09-13T15:00:01.448303+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.933789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.933789+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:48:08.137668+0000","scrub_duration":0.0011416200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":889,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698393+0000","last_change":"2024-09-13T14:34:46.559486+0000","last_active":"2024-09-13T15:00:03.698393+0000","last_peered":"2024-09-13T15:00:03.698393+0000","last_clean":"2024-09-13T15:00:03.698393+0000","last_became_active":"2024-09-13T14:34:46.559222+0000","last_became_peered":"2024-09-13T14:34:46.559222+0000","last_unstale":"2024-09-13T15:00:03.698393+0000","last_undegraded":"2024-09-13T15:00:03.698393+0000","last_fullsized":"2024-09-13T15:00:03.698393+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:31.997018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":236,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865071+0000","last_change":"2024-09-13T14:53:08.769698+0000","last_active":"2024-09-13T15:00:00.865071+0000","last_peered":"2024-09-13T15:00:00.865071+0000","last_clean":"2024-09-13T15:00:00.865071+0000","last_became_active":"2024-09-13T14:53:08.769260+0000","last_became_peered":"2024-09-13T14:53:08.769260+0000","last_unstale":"2024-09-13T15:00:00.865071+0000","last_undegraded":"2024-09-13T15:00:00.865071+0000","last_fullsized":"2024-09-13T15:00:00.865071+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:49:37.580959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'115","reported_seq":1133,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683108+0000","last_change":"2024-09-13T14:40:44.707358+0000","last_active":"2024-09-13T15:00:00.683108+0000","last_peered":"2024-09-13T15:00:00.683108+0000","last_clean":"2024-09-13T15:00:00.683108+0000","last_became_active":"2024-09-13T14:40:44.706890+0000","last_became_peered":"2024-09-13T14:40:44.706890+0000","last_unstale":"2024-09-13T15:00:00.683108+0000","last_undegraded":"2024-09-13T15:00:00.683108+0000","last_fullsized":"2024-09-13T15:00:00.683108+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:46.628685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056853700000000004,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":912,"num_read_kb":3139,"num_write":97,"num_write_kb":2230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"82'100","reported_seq":376,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682846+0000","last_change":"2024-09-13T14:56:00.644560+0000","last_active":"2024-09-13T15:00:00.682846+0000","last_peered":"2024-09-13T15:00:00.682846+0000","last_clean":"2024-09-13T15:00:00.682846+0000","last_became_active":"2024-09-13T14:55:54.621234+0000","last_became_peered":"2024-09-13T14:55:54.621234+0000","last_unstale":"2024-09-13T15:00:00.682846+0000","last_undegraded":"2024-09-13T15:00:00.682846+0000","last_fullsized":"2024-09-13T15:00:00.682846+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.644466+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.644466+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.375763+0000","scrub_duration":0.0093267220000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":554,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866615+0000","last_change":"2024-09-13T14:58:07.832480+0000","last_active":"2024-09-13T15:00:04.866615+0000","last_peered":"2024-09-13T15:00:04.866615+0000","last_clean":"2024-09-13T15:00:04.866615+0000","last_became_active":"2024-09-13T14:58:07.831527+0000","last_became_peered":"2024-09-13T14:58:07.831527+0000","last_unstale":"2024-09-13T15:00:04.866615+0000","last_undegraded":"2024-09-13T15:00:04.866615+0000","last_fullsized":"2024-09-13T15:00:04.866615+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:25.525827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":869,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698228+0000","last_change":"2024-09-13T14:34:46.602945+0000","last_active":"2024-09-13T15:00:03.698228+0000","last_peered":"2024-09-13T15:00:03.698228+0000","last_clean":"2024-09-13T15:00:03.698228+0000","last_became_active":"2024-09-13T14:34:46.602661+0000","last_became_peered":"2024-09-13T14:34:46.602661+0000","last_unstale":"2024-09-13T15:00:03.698228+0000","last_undegraded":"2024-09-13T15:00:03.698228+0000","last_fullsized":"2024-09-13T15:00:03.698228+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:54.775502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'113","reported_seq":928,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866999+0000","last_change":"2024-09-13T14:59:23.576669+0000","last_active":"2024-09-13T15:00:04.866999+0000","last_peered":"2024-09-13T15:00:04.866999+0000","last_clean":"2024-09-13T15:00:04.866999+0000","last_became_active":"2024-09-13T14:59:23.575951+0000","last_became_peered":"2024-09-13T14:59:23.575951+0000","last_unstale":"2024-09-13T15:00:04.866999+0000","last_undegraded":"2024-09-13T15:00:04.866999+0000","last_fullsized":"2024-09-13T15:00:04.866999+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:08:14.146814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064216200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":610,"num_read_kb":1067,"num_write":100,"num_write_kb":375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"82'100","reported_seq":689,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758751+0000","last_change":"2024-09-13T14:55:58.908251+0000","last_active":"2024-09-13T15:00:00.758751+0000","last_peered":"2024-09-13T15:00:00.758751+0000","last_clean":"2024-09-13T15:00:00.758751+0000","last_became_active":"2024-09-13T14:55:54.619910+0000","last_became_peered":"2024-09-13T14:55:54.619910+0000","last_unstale":"2024-09-13T15:00:00.758751+0000","last_undegraded":"2024-09-13T15:00:00.758751+0000","last_fullsized":"2024-09-13T15:00:00.758751+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:58.908152+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:58.908152+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:12:01.217177+0000","scrub_duration":0.0048598549999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865727+0000","last_change":"2024-09-13T14:53:08.772905+0000","last_active":"2024-09-13T15:00:04.865727+0000","last_peered":"2024-09-13T15:00:04.865727+0000","last_clean":"2024-09-13T15:00:04.865727+0000","last_became_active":"2024-09-13T14:53:08.772541+0000","last_became_peered":"2024-09-13T14:53:08.772541+0000","last_unstale":"2024-09-13T15:00:04.865727+0000","last_undegraded":"2024-09-13T15:00:04.865727+0000","last_fullsized":"2024-09-13T15:00:04.865727+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:19.468690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865541+0000","last_change":"2024-09-13T14:53:08.823773+0000","last_active":"2024-09-13T15:00:04.865541+0000","last_peered":"2024-09-13T15:00:04.865541+0000","last_clean":"2024-09-13T15:00:04.865541+0000","last_became_active":"2024-09-13T14:53:08.823336+0000","last_became_peered":"2024-09-13T14:53:08.823336+0000","last_unstale":"2024-09-13T15:00:04.865541+0000","last_undegraded":"2024-09-13T15:00:04.865541+0000","last_fullsized":"2024-09-13T15:00:04.865541+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:41.067229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'109","reported_seq":930,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758828+0000","last_change":"2024-09-13T14:53:09.821133+0000","last_active":"2024-09-13T15:00:00.758828+0000","last_peered":"2024-09-13T15:00:00.758828+0000","last_clean":"2024-09-13T15:00:00.758828+0000","last_became_active":"2024-09-13T14:53:09.820803+0000","last_became_peered":"2024-09-13T14:53:09.820803+0000","last_unstale":"2024-09-13T15:00:00.758828+0000","last_undegraded":"2024-09-13T15:00:00.758828+0000","last_fullsized":"2024-09-13T15:00:00.758828+0000","mapping_epoch":419,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":109,"log_dups_size":0,"ondisk_log_size":109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:26.046831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038132299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":662,"num_read_kb":1746,"num_write":96,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"82'100","reported_seq":761,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699204+0000","last_change":"2024-09-13T14:56:00.322316+0000","last_active":"2024-09-13T15:00:03.699204+0000","last_peered":"2024-09-13T15:00:03.699204+0000","last_clean":"2024-09-13T15:00:03.699204+0000","last_became_active":"2024-09-13T14:55:54.607589+0000","last_became_peered":"2024-09-13T14:55:54.607589+0000","last_unstale":"2024-09-13T15:00:03.699204+0000","last_undegraded":"2024-09-13T15:00:03.699204+0000","last_fullsized":"2024-09-13T15:00:03.699204+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.322230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.322230+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:58.559687+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":484,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448184+0000","last_change":"2024-09-13T14:40:26.259216+0000","last_active":"2024-09-13T15:00:01.448184+0000","last_peered":"2024-09-13T15:00:01.448184+0000","last_clean":"2024-09-13T15:00:01.448184+0000","last_became_active":"2024-09-13T14:40:26.258911+0000","last_became_peered":"2024-09-13T14:40:26.258911+0000","last_unstale":"2024-09-13T15:00:01.448184+0000","last_undegraded":"2024-09-13T15:00:01.448184+0000","last_fullsized":"2024-09-13T15:00:01.448184+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:20.883037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":882,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865450+0000","last_change":"2024-09-13T14:40:44.702004+0000","last_active":"2024-09-13T15:00:04.865450+0000","last_peered":"2024-09-13T15:00:04.865450+0000","last_clean":"2024-09-13T15:00:04.865450+0000","last_became_active":"2024-09-13T14:40:44.701692+0000","last_became_peered":"2024-09-13T14:40:44.701692+0000","last_unstale":"2024-09-13T15:00:04.865450+0000","last_undegraded":"2024-09-13T15:00:04.865450+0000","last_fullsized":"2024-09-13T15:00:04.865450+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:18.545904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'174","reported_seq":1042,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683039+0000","last_change":"2024-09-13T14:40:44.708783+0000","last_active":"2024-09-13T15:00:00.683039+0000","last_peered":"2024-09-13T15:00:00.683039+0000","last_clean":"2024-09-13T15:00:00.683039+0000","last_became_active":"2024-09-13T14:40:44.705748+0000","last_became_peered":"2024-09-13T14:40:44.705748+0000","last_unstale":"2024-09-13T15:00:00.683039+0000","last_undegraded":"2024-09-13T15:00:00.683039+0000","last_fullsized":"2024-09-13T15:00:00.683039+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:12:25.283940+0000","last_clean_scrub_stamp":"2024-09-13T14:12:25.283940+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:54.412998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055124999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":968,"num_read_kb":857,"num_write":148,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'100","reported_seq":660,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758921+0000","last_change":"2024-09-13T14:55:57.906683+0000","last_active":"2024-09-13T15:00:00.758921+0000","last_peered":"2024-09-13T15:00:00.758921+0000","last_clean":"2024-09-13T15:00:00.758921+0000","last_became_active":"2024-09-13T14:55:54.613915+0000","last_became_peered":"2024-09-13T14:55:54.613915+0000","last_unstale":"2024-09-13T15:00:00.758921+0000","last_undegraded":"2024-09-13T15:00:00.758921+0000","last_fullsized":"2024-09-13T15:00:00.758921+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:55:57.906630+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:55:57.906630+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:03.677753+0000","scrub_duration":0.001255616,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":890,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699326+0000","last_change":"2024-09-13T14:40:26.268658+0000","last_active":"2024-09-13T15:00:03.699326+0000","last_peered":"2024-09-13T15:00:03.699326+0000","last_clean":"2024-09-13T15:00:03.699326+0000","last_became_active":"2024-09-13T14:40:26.267603+0000","last_became_peered":"2024-09-13T14:40:26.267603+0000","last_unstale":"2024-09-13T15:00:03.699326+0000","last_undegraded":"2024-09-13T15:00:03.699326+0000","last_fullsized":"2024-09-13T15:00:03.699326+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:07.173229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":929,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865681+0000","last_change":"2024-09-13T14:34:46.583448+0000","last_active":"2024-09-13T15:00:04.865681+0000","last_peered":"2024-09-13T15:00:04.865681+0000","last_clean":"2024-09-13T15:00:04.865681+0000","last_became_active":"2024-09-13T14:34:46.582686+0000","last_became_peered":"2024-09-13T14:34:46.582686+0000","last_unstale":"2024-09-13T15:00:04.865681+0000","last_undegraded":"2024-09-13T15:00:04.865681+0000","last_fullsized":"2024-09-13T15:00:04.865681+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:38.133347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"82'100","reported_seq":356,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683039+0000","last_change":"2024-09-13T14:58:41.174258+0000","last_active":"2024-09-13T15:00:00.683039+0000","last_peered":"2024-09-13T15:00:00.683039+0000","last_clean":"2024-09-13T15:00:00.683039+0000","last_became_active":"2024-09-13T14:58:29.156586+0000","last_became_peered":"2024-09-13T14:58:29.156586+0000","last_unstale":"2024-09-13T15:00:00.683039+0000","last_undegraded":"2024-09-13T15:00:00.683039+0000","last_fullsized":"2024-09-13T15:00:00.683039+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:41.174201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:41.174201+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:22.118266+0000","scrub_duration":0.001866541,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865367+0000","last_change":"2024-09-13T14:40:26.265041+0000","last_active":"2024-09-13T15:00:00.865367+0000","last_peered":"2024-09-13T15:00:00.865367+0000","last_clean":"2024-09-13T15:00:00.865367+0000","last_became_active":"2024-09-13T14:40:26.263091+0000","last_became_peered":"2024-09-13T14:40:26.263091+0000","last_unstale":"2024-09-13T15:00:00.865367+0000","last_undegraded":"2024-09-13T15:00:00.865367+0000","last_fullsized":"2024-09-13T15:00:00.865367+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:35.970207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864614+0000","last_change":"2024-09-13T14:40:26.255401+0000","last_active":"2024-09-13T15:00:00.864614+0000","last_peered":"2024-09-13T15:00:00.864614+0000","last_clean":"2024-09-13T15:00:00.864614+0000","last_became_active":"2024-09-13T14:40:26.255062+0000","last_became_peered":"2024-09-13T14:40:26.255062+0000","last_unstale":"2024-09-13T15:00:00.864614+0000","last_undegraded":"2024-09-13T15:00:00.864614+0000","last_fullsized":"2024-09-13T15:00:00.864614+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:50:12.839019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"82'100","reported_seq":784,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.699096+0000","last_change":"2024-09-13T14:58:36.631751+0000","last_active":"2024-09-13T15:00:03.699096+0000","last_peered":"2024-09-13T15:00:03.699096+0000","last_clean":"2024-09-13T15:00:03.699096+0000","last_became_active":"2024-09-13T14:58:29.163891+0000","last_became_peered":"2024-09-13T14:58:29.163891+0000","last_unstale":"2024-09-13T15:00:03.699096+0000","last_undegraded":"2024-09-13T15:00:03.699096+0000","last_fullsized":"2024-09-13T15:00:03.699096+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.631685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.631685+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:37.084679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865335+0000","last_change":"2024-09-13T14:40:45.757723+0000","last_active":"2024-09-13T15:00:00.865335+0000","last_peered":"2024-09-13T15:00:00.865335+0000","last_clean":"2024-09-13T15:00:00.865335+0000","last_became_active":"2024-09-13T14:40:45.757112+0000","last_became_peered":"2024-09-13T14:40:45.757112+0000","last_unstale":"2024-09-13T15:00:00.865335+0000","last_undegraded":"2024-09-13T15:00:00.865335+0000","last_fullsized":"2024-09-13T15:00:00.865335+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:57.606688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.447855+0000","last_change":"2024-09-13T14:40:26.264023+0000","last_active":"2024-09-13T15:00:01.447855+0000","last_peered":"2024-09-13T15:00:01.447855+0000","last_clean":"2024-09-13T15:00:01.447855+0000","last_became_active":"2024-09-13T14:40:26.263572+0000","last_became_peered":"2024-09-13T14:40:26.263572+0000","last_unstale":"2024-09-13T15:00:01.447855+0000","last_undegraded":"2024-09-13T15:00:01.447855+0000","last_fullsized":"2024-09-13T15:00:01.447855+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:48.567932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"82'100","reported_seq":363,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682964+0000","last_change":"2024-09-13T14:58:42.127362+0000","last_active":"2024-09-13T15:00:00.682964+0000","last_peered":"2024-09-13T15:00:00.682964+0000","last_clean":"2024-09-13T15:00:00.682964+0000","last_became_active":"2024-09-13T14:58:29.160077+0000","last_became_peered":"2024-09-13T14:58:29.160077+0000","last_unstale":"2024-09-13T15:00:00.682964+0000","last_undegraded":"2024-09-13T15:00:00.682964+0000","last_fullsized":"2024-09-13T15:00:00.682964+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:42.127256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:42.127256+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:54.811181+0000","scrub_duration":0.0018624659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865094+0000","last_change":"2024-09-13T14:40:26.255891+0000","last_active":"2024-09-13T15:00:00.865094+0000","last_peered":"2024-09-13T15:00:00.865094+0000","last_clean":"2024-09-13T15:00:00.865094+0000","last_became_active":"2024-09-13T14:40:26.254289+0000","last_became_peered":"2024-09-13T14:40:26.254289+0000","last_unstale":"2024-09-13T15:00:00.865094+0000","last_undegraded":"2024-09-13T15:00:00.865094+0000","last_fullsized":"2024-09-13T15:00:00.865094+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.539061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"64'2","reported_seq":958,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865468+0000","last_change":"2024-09-13T14:53:08.868533+0000","last_active":"2024-09-13T15:00:04.865468+0000","last_peered":"2024-09-13T15:00:04.865468+0000","last_clean":"2024-09-13T15:00:04.865468+0000","last_became_active":"2024-09-13T14:53:08.785406+0000","last_became_peered":"2024-09-13T14:53:08.785406+0000","last_unstale":"2024-09-13T15:00:04.865468+0000","last_undegraded":"2024-09-13T15:00:04.865468+0000","last_fullsized":"2024-09-13T15:00:04.865468+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:45.014830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"82'100","reported_seq":649,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758701+0000","last_change":"2024-09-13T14:58:38.479706+0000","last_active":"2024-09-13T15:00:00.758701+0000","last_peered":"2024-09-13T15:00:00.758701+0000","last_clean":"2024-09-13T15:00:00.758701+0000","last_became_active":"2024-09-13T14:58:29.163808+0000","last_became_peered":"2024-09-13T14:58:29.163808+0000","last_unstale":"2024-09-13T15:00:00.758701+0000","last_undegraded":"2024-09-13T15:00:00.758701+0000","last_fullsized":"2024-09-13T15:00:00.758701+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:38.479649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:38.479649+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:45.827480+0000","scrub_duration":0.0015113780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866266+0000","last_change":"2024-09-13T14:53:08.773133+0000","last_active":"2024-09-13T15:00:04.866266+0000","last_peered":"2024-09-13T15:00:04.866266+0000","last_clean":"2024-09-13T15:00:04.866266+0000","last_became_active":"2024-09-13T14:53:08.772149+0000","last_became_peered":"2024-09-13T14:53:08.772149+0000","last_unstale":"2024-09-13T15:00:04.866266+0000","last_undegraded":"2024-09-13T15:00:04.866266+0000","last_fullsized":"2024-09-13T15:00:04.866266+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:38:20.597442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":446,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683117+0000","last_change":"2024-09-13T14:40:44.702597+0000","last_active":"2024-09-13T15:00:00.683117+0000","last_peered":"2024-09-13T15:00:00.683117+0000","last_clean":"2024-09-13T15:00:00.683117+0000","last_became_active":"2024-09-13T14:40:44.702323+0000","last_became_peered":"2024-09-13T14:40:44.702323+0000","last_unstale":"2024-09-13T15:00:00.683117+0000","last_undegraded":"2024-09-13T15:00:00.683117+0000","last_fullsized":"2024-09-13T15:00:00.683117+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:10.520384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"82'100","reported_seq":91,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.865194+0000","last_change":"2024-09-13T14:58:43.396350+0000","last_active":"2024-09-13T15:00:00.865194+0000","last_peered":"2024-09-13T15:00:00.865194+0000","last_clean":"2024-09-13T15:00:00.865194+0000","last_became_active":"2024-09-13T14:58:29.161667+0000","last_became_peered":"2024-09-13T14:58:29.161667+0000","last_unstale":"2024-09-13T15:00:00.865194+0000","last_undegraded":"2024-09-13T15:00:00.865194+0000","last_fullsized":"2024-09-13T15:00:00.865194+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:43.396294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:43.396294+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:08.269127+0000","scrub_duration":0.0013557140000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":498,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448162+0000","last_change":"2024-09-13T14:34:46.570224+0000","last_active":"2024-09-13T15:00:01.448162+0000","last_peered":"2024-09-13T15:00:01.448162+0000","last_clean":"2024-09-13T15:00:01.448162+0000","last_became_active":"2024-09-13T14:34:46.567755+0000","last_became_peered":"2024-09-13T14:34:46.567755+0000","last_unstale":"2024-09-13T15:00:01.448162+0000","last_undegraded":"2024-09-13T15:00:01.448162+0000","last_fullsized":"2024-09-13T15:00:01.448162+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:52:14.986747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759272+0000","last_change":"2024-09-13T14:40:44.712748+0000","last_active":"2024-09-13T15:00:00.759272+0000","last_peered":"2024-09-13T15:00:00.759272+0000","last_clean":"2024-09-13T15:00:00.759272+0000","last_became_active":"2024-09-13T14:40:44.712299+0000","last_became_peered":"2024-09-13T14:40:44.712299+0000","last_unstale":"2024-09-13T15:00:00.759272+0000","last_undegraded":"2024-09-13T15:00:00.759272+0000","last_fullsized":"2024-09-13T15:00:00.759272+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:38.738267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"82'100","reported_seq":546,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.447501+0000","last_change":"2024-09-13T14:56:02.698903+0000","last_active":"2024-09-13T15:00:01.447501+0000","last_peered":"2024-09-13T15:00:01.447501+0000","last_clean":"2024-09-13T15:00:01.447501+0000","last_became_active":"2024-09-13T14:56:00.691242+0000","last_became_peered":"2024-09-13T14:56:00.691242+0000","last_unstale":"2024-09-13T15:00:01.447501+0000","last_undegraded":"2024-09-13T15:00:01.447501+0000","last_fullsized":"2024-09-13T15:00:01.447501+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:02.698817+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:02.698817+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:50:11.424244+0000","scrub_duration":0.0077410969999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.864851+0000","last_change":"2024-09-13T14:40:26.264833+0000","last_active":"2024-09-13T15:00:00.864851+0000","last_peered":"2024-09-13T15:00:00.864851+0000","last_clean":"2024-09-13T15:00:00.864851+0000","last_became_active":"2024-09-13T14:40:26.264355+0000","last_became_peered":"2024-09-13T14:40:26.264355+0000","last_unstale":"2024-09-13T15:00:00.864851+0000","last_undegraded":"2024-09-13T15:00:00.864851+0000","last_fullsized":"2024-09-13T15:00:00.864851+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:18.932584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":529,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759318+0000","last_change":"2024-09-13T14:40:26.265814+0000","last_active":"2024-09-13T15:00:00.759318+0000","last_peered":"2024-09-13T15:00:00.759318+0000","last_clean":"2024-09-13T15:00:00.759318+0000","last_became_active":"2024-09-13T14:40:26.255031+0000","last_became_peered":"2024-09-13T14:40:26.255031+0000","last_unstale":"2024-09-13T15:00:00.759318+0000","last_undegraded":"2024-09-13T15:00:00.759318+0000","last_fullsized":"2024-09-13T15:00:00.759318+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:06.051650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"82'100","reported_seq":821,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866163+0000","last_change":"2024-09-13T14:58:36.861034+0000","last_active":"2024-09-13T15:00:04.866163+0000","last_peered":"2024-09-13T15:00:04.866163+0000","last_clean":"2024-09-13T15:00:04.866163+0000","last_became_active":"2024-09-13T14:58:29.165260+0000","last_became_peered":"2024-09-13T14:58:29.165260+0000","last_unstale":"2024-09-13T15:00:04.866163+0000","last_undegraded":"2024-09-13T15:00:04.866163+0000","last_fullsized":"2024-09-13T15:00:04.866163+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:36.860981+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:36.860981+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:35.870906+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":430,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682804+0000","last_change":"2024-09-13T14:40:44.702849+0000","last_active":"2024-09-13T15:00:00.682804+0000","last_peered":"2024-09-13T15:00:00.682804+0000","last_clean":"2024-09-13T15:00:00.682804+0000","last_became_active":"2024-09-13T14:40:44.702586+0000","last_became_peered":"2024-09-13T14:40:44.702586+0000","last_unstale":"2024-09-13T15:00:00.682804+0000","last_undegraded":"2024-09-13T15:00:00.682804+0000","last_fullsized":"2024-09-13T15:00:00.682804+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:41.481418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":539,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759353+0000","last_change":"2024-09-13T14:53:08.877465+0000","last_active":"2024-09-13T15:00:00.759353+0000","last_peered":"2024-09-13T15:00:00.759353+0000","last_clean":"2024-09-13T15:00:00.759353+0000","last_became_active":"2024-09-13T14:53:08.870551+0000","last_became_peered":"2024-09-13T14:53:08.870551+0000","last_unstale":"2024-09-13T15:00:00.759353+0000","last_undegraded":"2024-09-13T15:00:00.759353+0000","last_fullsized":"2024-09-13T15:00:00.759353+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:14.525127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"82'100","reported_seq":91,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.759070+0000","last_change":"2024-09-13T14:58:44.446102+0000","last_active":"2024-09-13T15:00:00.759070+0000","last_peered":"2024-09-13T15:00:00.759070+0000","last_clean":"2024-09-13T15:00:00.759070+0000","last_became_active":"2024-09-13T14:58:29.164312+0000","last_became_peered":"2024-09-13T14:58:29.164312+0000","last_unstale":"2024-09-13T15:00:00.759070+0000","last_undegraded":"2024-09-13T15:00:00.759070+0000","last_fullsized":"2024-09-13T15:00:00.759070+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:58:44.445993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:58:44.445993+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:34.621592+0000","scrub_duration":0.0014128000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":547,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:01.448017+0000","last_change":"2024-09-13T14:53:08.829604+0000","last_active":"2024-09-13T15:00:01.448017+0000","last_peered":"2024-09-13T15:00:01.448017+0000","last_clean":"2024-09-13T15:00:01.448017+0000","last_became_active":"2024-09-13T14:53:08.768805+0000","last_became_peered":"2024-09-13T14:53:08.768805+0000","last_unstale":"2024-09-13T15:00:01.448017+0000","last_undegraded":"2024-09-13T15:00:01.448017+0000","last_fullsized":"2024-09-13T15:00:01.448017+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:09:43.164993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866148+0000","last_change":"2024-09-13T14:34:46.565368+0000","last_active":"2024-09-13T15:00:04.866148+0000","last_peered":"2024-09-13T15:00:04.866148+0000","last_clean":"2024-09-13T15:00:04.866148+0000","last_became_active":"2024-09-13T14:34:46.564941+0000","last_became_peered":"2024-09-13T14:34:46.564941+0000","last_unstale":"2024-09-13T15:00:04.866148+0000","last_undegraded":"2024-09-13T15:00:04.866148+0000","last_fullsized":"2024-09-13T15:00:04.866148+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:07.441857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"82'100","reported_seq":71,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683002+0000","last_change":"2024-09-13T14:56:01.635272+0000","last_active":"2024-09-13T15:00:00.683002+0000","last_peered":"2024-09-13T15:00:00.683002+0000","last_clean":"2024-09-13T15:00:00.683002+0000","last_became_active":"2024-09-13T14:56:00.667926+0000","last_became_peered":"2024-09-13T14:56:00.667926+0000","last_unstale":"2024-09-13T15:00:00.683002+0000","last_undegraded":"2024-09-13T15:00:00.683002+0000","last_fullsized":"2024-09-13T15:00:00.683002+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.635189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.635189+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:28.334884+0000","scrub_duration":0.010355512000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":485,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758973+0000","last_change":"2024-09-13T14:40:26.257399+0000","last_active":"2024-09-13T15:00:00.758973+0000","last_peered":"2024-09-13T15:00:00.758973+0000","last_clean":"2024-09-13T15:00:00.758973+0000","last_became_active":"2024-09-13T14:40:26.257070+0000","last_became_peered":"2024-09-13T14:40:26.257070+0000","last_unstale":"2024-09-13T15:00:00.758973+0000","last_undegraded":"2024-09-13T15:00:00.758973+0000","last_fullsized":"2024-09-13T15:00:00.758973+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:27.425589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866762+0000","last_change":"2024-09-13T14:53:08.803722+0000","last_active":"2024-09-13T15:00:04.866762+0000","last_peered":"2024-09-13T15:00:04.866762+0000","last_clean":"2024-09-13T15:00:04.866762+0000","last_became_active":"2024-09-13T14:53:08.803425+0000","last_became_peered":"2024-09-13T14:53:08.803425+0000","last_unstale":"2024-09-13T15:00:04.866762+0000","last_undegraded":"2024-09-13T15:00:04.866762+0000","last_fullsized":"2024-09-13T15:00:04.866762+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:24:05.477497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"82'100","reported_seq":767,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:03.698942+0000","last_change":"2024-09-13T14:56:01.351509+0000","last_active":"2024-09-13T15:00:03.698942+0000","last_peered":"2024-09-13T15:00:03.698942+0000","last_clean":"2024-09-13T15:00:03.698942+0000","last_became_active":"2024-09-13T14:56:00.675249+0000","last_became_peered":"2024-09-13T14:56:00.675249+0000","last_unstale":"2024-09-13T15:00:03.698942+0000","last_undegraded":"2024-09-13T15:00:03.698942+0000","last_fullsized":"2024-09-13T15:00:03.698942+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:01.351298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:01.351298+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:36.329389+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":213,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866148+0000","last_change":"2024-09-13T14:53:08.798674+0000","last_active":"2024-09-13T15:00:04.866148+0000","last_peered":"2024-09-13T15:00:04.866148+0000","last_clean":"2024-09-13T15:00:04.866148+0000","last_became_active":"2024-09-13T14:53:08.797852+0000","last_became_peered":"2024-09-13T14:53:08.797852+0000","last_unstale":"2024-09-13T15:00:04.866148+0000","last_undegraded":"2024-09-13T15:00:04.866148+0000","last_fullsized":"2024-09-13T15:00:04.866148+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:00:09.370841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":445,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.683407+0000","last_change":"2024-09-13T14:40:44.705177+0000","last_active":"2024-09-13T15:00:00.683407+0000","last_peered":"2024-09-13T15:00:00.683407+0000","last_clean":"2024-09-13T15:00:00.683407+0000","last_became_active":"2024-09-13T14:40:44.704861+0000","last_became_peered":"2024-09-13T14:40:44.704861+0000","last_unstale":"2024-09-13T15:00:00.683407+0000","last_undegraded":"2024-09-13T15:00:00.683407+0000","last_fullsized":"2024-09-13T15:00:00.683407+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:51:29.585203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"82'100","reported_seq":71,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.682697+0000","last_change":"2024-09-13T14:56:04.574886+0000","last_active":"2024-09-13T15:00:00.682697+0000","last_peered":"2024-09-13T15:00:00.682697+0000","last_clean":"2024-09-13T15:00:00.682697+0000","last_became_active":"2024-09-13T14:56:00.674692+0000","last_became_peered":"2024-09-13T14:56:00.674692+0000","last_unstale":"2024-09-13T15:00:00.682697+0000","last_undegraded":"2024-09-13T15:00:00.682697+0000","last_fullsized":"2024-09-13T15:00:00.682697+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:04.574787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:09:56.715529+0000","last_clean_scrub_stamp":"2024-09-13T14:56:04.574787+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:40:30.673194+0000","scrub_duration":0.0020770139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":516,"reported_epoch":517,"state":"active+clean","last_fresh":"2024-09-13T15:00:00.758956+0000","last_change":"2024-09-13T14:40:26.274006+0000","last_active":"2024-09-13T15:00:00.758956+0000","last_peered":"2024-09-13T15:00:00.758956+0000","last_clean":"2024-09-13T15:00:00.758956+0000","last_became_active":"2024-09-13T14:40:26.258237+0000","last_became_peered":"2024-09-13T14:40:26.258237+0000","last_unstale":"2024-09-13T15:00:00.758956+0000","last_undegraded":"2024-09-13T15:00:00.758956+0000","last_fullsized":"2024-09-13T15:00:00.758956+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:40.105974+0000","last_clean_scrub_stamp":"2024-09-13T14:13:40.105974+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:05.745352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":926,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.866273+0000","last_change":"2024-09-13T14:34:46.568268+0000","last_active":"2024-09-13T15:00:04.866273+0000","last_peered":"2024-09-13T15:00:04.866273+0000","last_clean":"2024-09-13T15:00:04.866273+0000","last_became_active":"2024-09-13T14:34:46.566761+0000","last_became_peered":"2024-09-13T14:34:46.566761+0000","last_unstale":"2024-09-13T15:00:04.866273+0000","last_undegraded":"2024-09-13T15:00:04.866273+0000","last_fullsized":"2024-09-13T15:00:04.866273+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:13:41.111573+0000","last_clean_scrub_stamp":"2024-09-13T14:13:41.111573+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:12.134991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"82'100","reported_seq":377,"reported_epoch":518,"state":"active+clean","last_fresh":"2024-09-13T15:00:04.865752+0000","last_change":"2024-09-13T14:56:00.831620+0000","last_active":"2024-09-13T15:00:04.865752+0000","last_peered":"2024-09-13T15:00:04.865752+0000","last_clean":"2024-09-13T15:00:04.865752+0000","last_became_active":"2024-09-13T14:56:00.676063+0000","last_became_peered":"2024-09-13T14:56:00.676063+0000","last_unstale":"2024-09-13T15:00:04.865752+0000","last_undegraded":"2024-09-13T15:00:04.865752+0000","last_fullsized":"2024-09-13T15:00:04.865752+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":463,"last_epoch_clean":467,"parent":"0.0","parent_split_bits":5,"last_scrub":"82'100","last_scrub_stamp":"2024-09-13T14:56:00.831575+0000","last_deep_scrub":"82'100","last_deep_scrub_stamp":"2024-09-13T14:49:47.727247+0000","last_clean_scrub_stamp":"2024-09-13T14:56:00.831575+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:20.751784+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":5650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26545,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5968,"num_read_kb":12981,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398098,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915340,"kb_used_data":11008,"kb_used_omap":1,"kb_used_meta":904318,"kb_avail":92829812,"statfs":{"total":95995035648,"available":95057727488,"internally_reserved":0,"allocated":11272192,"data_stored":8579930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926021976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594326,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905592,"kb_used_data":6956,"kb_used_omap":1,"kb_used_meta":898622,"kb_avail":92839560,"statfs":{"total":95995035648,"available":95067709440,"internally_reserved":0,"allocated":7122944,"data_stored":4449916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":920189272},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":316,"seq":1357209665780,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928756,"kb_used_data":6888,"kb_used_omap":9,"kb_used_meta":921846,"kb_avail":92816396,"statfs":{"total":95995035648,"available":95043989504,"internally_reserved":0,"allocated":7053312,"data_stored":4380687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10040,"internal_metadata":943970504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986782,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910660,"kb_used_data":6648,"kb_used_omap":1,"kb_used_meta":903998,"kb_avail":92834492,"statfs":{"total":95995035648,"available":95062519808,"internally_reserved":0,"allocated":6807552,"data_stored":4122868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925694296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":101,"seq":433791697342,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146416,"kb_used_data":6948,"kb_used_omap":4,"kb_used_meta":1139451,"kb_avail":92598736,"statfs":{"total":95995035648,"available":94821105664,"internally_reserved":0,"allocated":7114752,"data_stored":4445760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5045,"internal_metadata":1166797899},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":96,"seq":412316860868,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899768,"kb_used_data":6892,"kb_used_omap":8,"kb_used_meta":892855,"kb_avail":92845384,"statfs":{"total":95995035648,"available":95073673216,"internally_reserved":0,"allocated":7057408,"data_stored":4381282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8372,"internal_metadata":914284364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":244,"seq":1047972020539,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":680560,"kb_used_data":10724,"kb_used_omap":9,"kb_used_meta":669814,"kb_avail":93064592,"statfs":{"total":95995035648,"available":95298142208,"internally_reserved":0,"allocated":10981376,"data_stored":8312305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9403,"internal_metadata":685890373},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":86,"seq":369367187930,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036464,"kb_used_data":11044,"kb_used_omap":7,"kb_used_meta":1025400,"kb_avail":92708688,"statfs":{"total":95995035648,"available":94933696512,"internally_reserved":0,"allocated":11309056,"data_stored":8640251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":1050009931},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4006,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6157,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6049,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2662,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3341,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8336,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:00:57.509 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:00:57.509 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:00:57.687 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=83878f5762b540d0b642848dc5bc46e6 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:00:57.690 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T15:00:57.701 INFO:tasks.cephadm:Teardown begin 2024-09-13T15:00:57.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:00:57.715 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:57 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: ::ffff:172.21.15.189 - - [13/Sep/2024:15:00:57] "GET /metrics HTTP/1.1" 200 34299 "" "Prometheus/2.51.0" 2024-09-13T15:00:57.752 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:00:57.796 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T15:00:57.797 DEBUG:teuthology.orchestra.run.smithi017:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:00:57.815 DEBUG:teuthology.orchestra.run.smithi189:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:00:57.854 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T15:00:57.854 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T15:00:57.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T15:00:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:57 smithi189 ceph-mon[90992]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:58.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:00:57 smithi189 ceph-mon[90992]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:58.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:57 smithi017 ceph-mon[107556]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:58.155 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:57 smithi017 ceph-mon[107556]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:58.155 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:57 smithi017 ceph-mon[103001]: from='client.37635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:58.155 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:57 smithi017 ceph-mon[103001]: from='mgr.34107 172.21.15.17:0/2768149622' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:58.155 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:57 smithi017 systemd[1]: Stopping Ceph mon.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:00:58.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T15:00:58.152+0000 7f0b451cc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:00:58.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a[102976]: 2024-09-13T15:00:58.152+0000 7f0b451cc640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T15:00:58.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 podman[416329]: 2024-09-13 15:00:58.357327613 +0000 UTC m=+0.301646891 container died c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T15:00:58.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE Bus STOPPING 2024-09-13T15:00:58.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:00:58.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE Bus STOPPED 2024-09-13T15:00:58.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE Bus STARTING 2024-09-13T15:00:58.752 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE Serving on http://:::9283 2024-09-13T15:00:58.752 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:00:58 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: [13/Sep/2024:15:00:58] ENGINE Bus STARTED 2024-09-13T15:00:59.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 podman[416329]: 2024-09-13 15:00:58.751270495 +0000 UTC m=+0.695589770 container cleanup c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T15:00:59.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 bash[416329]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T15:00:59.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 13 15:00:58 smithi017 podman[416346]: 2024-09-13 15:00:58.936371782 +0000 UTC m=+0.574824201 container remove c2de9b169292cd83a1eeac5675d8402e5ab392f67b1b02cf39ba6a87c72fb746 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T15:00:59.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T15:00:59.197 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:00:59.197 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T15:00:59.197 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T15:00:59.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.c 2024-09-13T15:00:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:59 smithi017 systemd[1]: Stopping Ceph mon.c for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:00:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:59 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c[107533]: 2024-09-13T15:00:59.549+0000 7f77dae35640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:00:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:59 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c[107533]: 2024-09-13T15:00:59.549+0000 7f77dae35640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:00:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:00:59 smithi017 podman[416473]: 2024-09-13 15:00:59.67261227 +0000 UTC m=+0.243634085 container died c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T15:01:00.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:01:00 smithi017 podman[416473]: 2024-09-13 15:01:00.181521459 +0000 UTC m=+0.752543316 container cleanup c1ddeb604caca4b8b8bc0b6c13c7c8a2d00c9cda41edb0dfb96d71754a67925a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T15:01:00.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 13 15:01:00 smithi017 bash[416473]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T15:01:00.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"523f52e4-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:00.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T15:01:00.590 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:00.591 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T15:01:00.591 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T15:01:00.591 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.b 2024-09-13T15:01:00.924 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:00 smithi189 systemd[1]: Stopping Ceph mon.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:01.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b[90969]: 2024-09-13T15:01:00.922+0000 7f5342097640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:01.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b[90969]: 2024-09-13T15:01:00.922+0000 7f5342097640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:01:01.256 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:01 smithi189 podman[268215]: 2024-09-13 15:01:01.168195098 +0000 UTC m=+0.361461594 container died 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T15:01:02.074 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:01 smithi189 podman[268215]: 2024-09-13 15:01:01.762922622 +0000 UTC m=+0.956189117 container cleanup 37d58338d5698792b0cd15cbb4c2d5c6083eec829c2d059002aec6d2eba856f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T15:01:02.074 INFO:journalctl@ceph.mon.b.smithi189.stdout:Sep 13 15:01:01 smithi189 bash[268215]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T15:01:02.199 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T15:01:02.241 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:02.241 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T15:01:02.242 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T15:01:02.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T15:01:02.549 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:01:02 smithi017 systemd[1]: Stopping Ceph mgr.y for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:02.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:01:02 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y[93261]: 2024-09-13T15:01:02.546+0000 7f914248b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:02.830 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:01:02 smithi017 podman[416612]: 2024-09-13 15:01:02.588742313 +0000 UTC m=+0.144730456 container died b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:03.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:01:03 smithi017 podman[416612]: 2024-09-13 15:01:03.045031644 +0000 UTC m=+0.601019789 container cleanup b59080328aea28e18d664c5f41137dc83325d6e2c2f8c62814e779a133040324 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T15:01:03.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 13 15:01:03 smithi017 bash[416612]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T15:01:03.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T15:01:03.576 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:03.577 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T15:01:03.577 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T15:01:03.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x 2024-09-13T15:01:03.902 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 15:01:03 smithi189 systemd[1]: Stopping Ceph mgr.x for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:04.170 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 15:01:03 smithi189 podman[268358]: 2024-09-13 15:01:03.901348015 +0000 UTC m=+0.136183746 container died e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T15:01:04.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 15:01:04 smithi189 podman[268358]: 2024-09-13 15:01:04.259772318 +0000 UTC m=+0.494608050 container cleanup e2be48bfd3a81f30096685ccd79b70d4b7a5c31794a639eb834798e1442fb0c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T15:01:04.506 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Sep 13 15:01:04 smithi189 bash[268358]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T15:01:04.720 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T15:01:04.755 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:04.755 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T15:01:04.755 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T15:01:04.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.0 2024-09-13T15:01:05.062 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:04 smithi017 systemd[1]: Stopping Ceph osd.0 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:05.450 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T15:01:05.060+0000 7f02e6be2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:05.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T15:01:05.060+0000 7f02e6be2640 -1 osd.0 518 *** Got signal Terminated *** 2024-09-13T15:01:05.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:05 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0[116038]: 2024-09-13T15:01:05.060+0000 7f02e6be2640 -1 osd.0 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:10.844 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:10 smithi017 podman[416743]: 2024-09-13 15:01:10.516333181 +0000 UTC m=+5.564745489 container died b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-13T15:01:11.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:11 smithi017 podman[416743]: 2024-09-13 15:01:11.0249861 +0000 UTC m=+6.073398408 container cleanup b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T15:01:11.201 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:11 smithi017 bash[416743]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T15:01:11.897 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:11 smithi017 podman[416765]: 2024-09-13 15:01:11.560417676 +0000 UTC m=+1.041662725 container remove b6c1265d945352ea16f7aa6f9338f59580a55c24f49858cc3c4ae6e8361f8bfe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T15:01:12.182 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:11 smithi017 podman[416857]: 2024-09-13 15:01:11.79950035 +0000 UTC m=+0.016735269 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:01:12.182 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:11 smithi017 podman[416857]: 2024-09-13 15:01:11.919852556 +0000 UTC m=+0.137087509 container create 3a6676a2223ac5fded545acb64608f1c752a564ebf4f2f2c09529af0039430fe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:12.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:12 smithi017 podman[416857]: 2024-09-13 15:01:12.181034213 +0000 UTC m=+0.398269121 container init 3a6676a2223ac5fded545acb64608f1c752a564ebf4f2f2c09529af0039430fe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T15:01:12.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:12 smithi017 podman[416857]: 2024-09-13 15:01:12.186645524 +0000 UTC m=+0.403880431 container start 3a6676a2223ac5fded545acb64608f1c752a564ebf4f2f2c09529af0039430fe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T15:01:12.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:12 smithi017 podman[416857]: 2024-09-13 15:01:12.306026236 +0000 UTC m=+0.523261222 container attach 3a6676a2223ac5fded545acb64608f1c752a564ebf4f2f2c09529af0039430fe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:13.402 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 13 15:01:13 smithi017 podman[416857]: 2024-09-13 15:01:13.06129328 +0000 UTC m=+1.278528194 container died 3a6676a2223ac5fded545acb64608f1c752a564ebf4f2f2c09529af0039430fe (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T15:01:13.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T15:01:13.883 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:13.883 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T15:01:13.883 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T15:01:13.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1 2024-09-13T15:01:14.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:13 smithi017 systemd[1]: Stopping Ceph osd.1 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:14.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T15:01:14.228+0000 7fcb2be6d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:14.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T15:01:14.228+0000 7fcb2be6d640 -1 osd.1 518 *** Got signal Terminated *** 2024-09-13T15:01:14.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:14 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1[215097]: 2024-09-13T15:01:14.228+0000 7fcb2be6d640 -1 osd.1 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:20.090 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:19 smithi017 podman[416987]: 2024-09-13 15:01:19.774117078 +0000 UTC m=+5.657022304 container died a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:20.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:20 smithi017 podman[416987]: 2024-09-13 15:01:20.254690496 +0000 UTC m=+6.137595718 container cleanup a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T15:01:20.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:20 smithi017 bash[416987]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T15:01:20.898 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:20 smithi017 podman[417008]: 2024-09-13 15:01:20.581504632 +0000 UTC m=+0.799035263 container remove a70d0d1e41c8efb15f4877fb027d56d90d560959e8127cee95960a2baa64c954 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T15:01:21.152 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:20 smithi017 podman[417100]: 2024-09-13 15:01:20.800322787 +0000 UTC m=+0.016252349 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:01:21.153 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:20 smithi017 podman[417100]: 2024-09-13 15:01:20.907515809 +0000 UTC m=+0.123445401 container create 3809f3ad0111b2b8736b5044f94c0632ac65e326dfded6fa54b1d4d166cd617c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T15:01:21.153 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:21 smithi017 podman[417100]: 2024-09-13 15:01:21.151382039 +0000 UTC m=+0.367311596 container init 3809f3ad0111b2b8736b5044f94c0632ac65e326dfded6fa54b1d4d166cd617c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T15:01:21.451 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:21 smithi017 podman[417100]: 2024-09-13 15:01:21.156828323 +0000 UTC m=+0.372757874 container start 3809f3ad0111b2b8736b5044f94c0632ac65e326dfded6fa54b1d4d166cd617c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T15:01:21.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:21 smithi017 podman[417100]: 2024-09-13 15:01:21.276691603 +0000 UTC m=+0.492621171 container attach 3809f3ad0111b2b8736b5044f94c0632ac65e326dfded6fa54b1d4d166cd617c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T15:01:21.452 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 13 15:01:21 smithi017 podman[417100]: 2024-09-13 15:01:21.352252163 +0000 UTC m=+0.568181713 container died 3809f3ad0111b2b8736b5044f94c0632ac65e326dfded6fa54b1d4d166cd617c (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T15:01:22.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T15:01:22.158 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:22.171 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T15:01:22.171 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T15:01:22.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.2 2024-09-13T15:01:22.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:22 smithi017 systemd[1]: Stopping Ceph osd.2 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:22.951 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T15:01:22.485+0000 7fdf05e95640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:22.951 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T15:01:22.485+0000 7fdf05e95640 -1 osd.2 518 *** Got signal Terminated *** 2024-09-13T15:01:22.951 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:22 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2[127336]: 2024-09-13T15:01:22.485+0000 7fdf05e95640 -1 osd.2 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:28.274 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:27 smithi017 podman[417230]: 2024-09-13 15:01:27.972699958 +0000 UTC m=+5.598841451 container died f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:01:28.694 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:28 smithi017 podman[417230]: 2024-09-13 15:01:28.450230655 +0000 UTC m=+6.076372149 container cleanup f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T15:01:28.694 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:28 smithi017 bash[417230]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T15:01:28.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:28 smithi017 podman[417253]: 2024-09-13 15:01:28.693350146 +0000 UTC m=+0.723463008 container remove f910c1d6db6afad13e1efbce5ee88bc539330a028607817d1e2d994912163093 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:29.286 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:28.935338356 +0000 UTC m=+0.016698477 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:01:29.286 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:29.069624515 +0000 UTC m=+0.150984671 container create 38d1e1bba76d787026ac199403114f53d3240097d5daf345d622e904848d0f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T15:01:29.286 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:29.280168411 +0000 UTC m=+0.361528522 container init 38d1e1bba76d787026ac199403114f53d3240097d5daf345d622e904848d0f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:01:29.286 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:29.285129959 +0000 UTC m=+0.366490072 container start 38d1e1bba76d787026ac199403114f53d3240097d5daf345d622e904848d0f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:29.701 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:29.388498597 +0000 UTC m=+0.469858709 container attach 38d1e1bba76d787026ac199403114f53d3240097d5daf345d622e904848d0f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:29.701 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 13 15:01:29 smithi017 podman[417348]: 2024-09-13 15:01:29.481760875 +0000 UTC m=+0.563120994 container died 38d1e1bba76d787026ac199403114f53d3240097d5daf345d622e904848d0f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:30.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T15:01:30.106 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:30.106 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T15:01:30.106 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T15:01:30.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:01:30.331 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:30 smithi017 systemd[1]: Stopping Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:30.700 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:30.454+0000 7fef63c25640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:30.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:30.454+0000 7fef63c25640 -1 osd.3 518 *** Got signal Terminated *** 2024-09-13T15:01:30.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:30.454+0000 7fef63c25640 -1 osd.3 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:31.200 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:30 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:30.748+0000 7fef6023a640 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.215309+0000 front 2024-09-13T15:01:05.215332+0000 (oldest deadline 2024-09-13T15:01:30.515316+0000) 2024-09-13T15:01:32.200 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:31 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:31.732+0000 7fef6023a640 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.215309+0000 front 2024-09-13T15:01:05.215332+0000 (oldest deadline 2024-09-13T15:01:30.515316+0000) 2024-09-13T15:01:32.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:32.006+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:32.507 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:32.129+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:33.200 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:32 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:32.745+0000 7fef6023a640 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.215309+0000 front 2024-09-13T15:01:05.215332+0000 (oldest deadline 2024-09-13T15:01:30.515316+0000) 2024-09-13T15:01:33.256 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:32 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:32.988+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:33.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:33 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:33.119+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:34.200 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:33 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:33.762+0000 7fef6023a640 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.215309+0000 front 2024-09-13T15:01:05.215332+0000 (oldest deadline 2024-09-13T15:01:30.515316+0000) 2024-09-13T15:01:34.418 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:34.029+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:34.419 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:34 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:34.083+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:35.200 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:34 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3[133348]: 2024-09-13T15:01:34.713+0000 7fef6023a640 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.215309+0000 front 2024-09-13T15:01:05.215332+0000 (oldest deadline 2024-09-13T15:01:30.515316+0000) 2024-09-13T15:01:35.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:35.060+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:35.506 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:35.079+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:35.506 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:35 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:35.106+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:36.382 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:36 smithi017 podman[417478]: 2024-09-13 15:01:36.087916646 +0000 UTC m=+5.752294493 container died 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:36.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:36.099+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:36.506 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:36.058+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:36.507 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:36.108+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:36.507 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:36 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:36.176+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:36.697 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:36 smithi017 podman[417478]: 2024-09-13 15:01:36.52876459 +0000 UTC m=+6.193142479 container cleanup 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:36.698 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:36 smithi017 bash[417478]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T15:01:36.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:36 smithi017 podman[417511]: 2024-09-13 15:01:36.696670676 +0000 UTC m=+0.605398841 container remove 44369002d276dc9e57d3780e1185276c17ae8c6d0c669d619efac74fb74316f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T15:01:37.417 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.00203763 +0000 UTC m=+0.016983529 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:01:37.417 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.131041337 +0000 UTC m=+0.145987287 container create f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T15:01:37.417 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.333074182 +0000 UTC m=+0.348020080 container init f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T15:01:37.417 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.338480269 +0000 UTC m=+0.353426168 container start f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T15:01:37.417 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.416226608 +0000 UTC m=+0.431172507 container attach f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T15:01:37.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:37 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:37.178+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:37.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:37 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:37.140+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:37.507 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:37 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:37.049+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:37.507 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:37 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:37.065+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:37.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:37 smithi017 podman[417604]: 2024-09-13 15:01:37.528984452 +0000 UTC m=+0.543930354 container died f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:01:38.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T15:01:38.111 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:38 smithi017 podman[417646]: 2024-09-13 15:01:38.069653752 +0000 UTC m=+0.530417316 container remove f817192fda686e655e089f545ec078731ff5eab4382ed64572810e01134ff4bf (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2) 2024-09-13T15:01:38.111 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:38 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T15:01:38.112 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:38 smithi017 systemd[1]: Stopped Ceph osd.3 for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:01:38.112 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 13 15:01:38 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.3.service: Consumed 37min 1.960s CPU time. 2024-09-13T15:01:38.127 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:38.128 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T15:01:38.128 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T15:01:38.128 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.4 2024-09-13T15:01:38.474 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:38.152+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:38.475 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:38.060+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:38.475 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:38.031+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:38.475 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:38 smithi189 systemd[1]: Stopping Ceph osd.4 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:38.475 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:38.187+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:38.476 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:38.472+0000 7ff87a323640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:38.476 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:38.472+0000 7ff87a323640 -1 osd.4 518 *** Got signal Terminated *** 2024-09-13T15:01:38.476 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:38 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:38.472+0000 7ff87a323640 -1 osd.4 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:39.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:39.167+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:39.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:39.224+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:39.506 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:39.089+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:39.507 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:39.018+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:40.256 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:40.207+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:40.257 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:40.154+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:40.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:40.103+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:40.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:39 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:39.981+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:41.234 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:41.108+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:41.234 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:41.126+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:41.234 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:40.960+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:41.234 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:40 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:40.960+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:41.505 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:41.232+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:41.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:41.232+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.573632+0000 front 2024-09-13T15:01:15.573467+0000 (oldest deadline 2024-09-13T15:01:40.273340+0000) 2024-09-13T15:01:42.254 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:42.070+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:42.254 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:42.085+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:42.255 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:41.922+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:42.255 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:41 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:41.922+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:42.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:42.252+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:42.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:42.252+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.573632+0000 front 2024-09-13T15:01:15.573467+0000 (oldest deadline 2024-09-13T15:01:40.273340+0000) 2024-09-13T15:01:43.232 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:43.071+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:43.233 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:43.115+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:43.233 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:43.115+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:43.233 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:42.897+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:43.233 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:42 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:42.897+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:43.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:43.231+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.673070+0000 front 2024-09-13T15:01:09.673007+0000 (oldest deadline 2024-09-13T15:01:35.572914+0000) 2024-09-13T15:01:43.506 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4[38586]: 2024-09-13T15:01:43.231+0000 7ff87713c640 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.573632+0000 front 2024-09-13T15:01:15.573467+0000 (oldest deadline 2024-09-13T15:01:40.273340+0000) 2024-09-13T15:01:44.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:44.067+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:44.256 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:44 smithi189 podman[268502]: 2024-09-13 15:01:44.132855856 +0000 UTC m=+5.796336031 container died a06c4fb486c57b6a4f7e7647dabde3c3f72965ad9e69ed1106dc77113d0ee318 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:01:44.257 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:44.102+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:44.257 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:44.102+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:44.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:43.940+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:44.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:43 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:43.940+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.122Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.124Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.126Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.126Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.127Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.258 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:01:44.127Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-13T15:01:44.756 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:44 smithi189 podman[268502]: 2024-09-13 15:01:44.629575041 +0000 UTC m=+6.293055258 container cleanup a06c4fb486c57b6a4f7e7647dabde3c3f72965ad9e69ed1106dc77113d0ee318 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True) 2024-09-13T15:01:44.756 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:44 smithi189 bash[268502]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T15:01:45.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:44.941+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:45.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:44 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:44.941+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:45.257 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:44 smithi189 podman[268527]: 2024-09-13 15:01:44.94799495 +0000 UTC m=+0.799468389 container remove a06c4fb486c57b6a4f7e7647dabde3c3f72965ad9e69ed1106dc77113d0ee318 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T15:01:45.257 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:45.099+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:45.257 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:45.099+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:45.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:45.030+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:45.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:45.030+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:45.552 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.161853186 +0000 UTC m=+0.017203691 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:01:45.553 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.274101002 +0000 UTC m=+0.129451503 container create d499f5168854639ae9eb4c48e4b4d0d46fa2ef417f26c969b3118ea74adfa035 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:01:45.553 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.460149156 +0000 UTC m=+0.315499674 container init d499f5168854639ae9eb4c48e4b4d0d46fa2ef417f26c969b3118ea74adfa035 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:01:45.553 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.465022221 +0000 UTC m=+0.320372731 container start d499f5168854639ae9eb4c48e4b4d0d46fa2ef417f26c969b3118ea74adfa035 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T15:01:45.553 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.551661263 +0000 UTC m=+0.407011775 container attach d499f5168854639ae9eb4c48e4b4d0d46fa2ef417f26c969b3118ea74adfa035 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2) 2024-09-13T15:01:46.105 INFO:journalctl@ceph.osd.4.smithi189.stdout:Sep 13 15:01:45 smithi189 podman[268620]: 2024-09-13 15:01:45.819164446 +0000 UTC m=+0.674514960 container died d499f5168854639ae9eb4c48e4b4d0d46fa2ef417f26c969b3118ea74adfa035 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T15:01:46.106 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:46.062+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:46.106 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:46.062+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:46.106 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:46.064+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:46.107 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:46.064+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:46.107 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:45.900+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:46.107 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:45 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:45.900+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:46.398 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T15:01:46.438 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:46.439 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T15:01:46.439 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T15:01:46.439 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5 2024-09-13T15:01:46.756 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 systemd[1]: Stopping Ceph osd.5 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:47.027 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:46.913+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:47.027 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:46.913+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:47.028 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:46.773+0000 7f3b8f0dc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:47.028 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:46.773+0000 7f3b8f0dc640 -1 osd.5 518 *** Got signal Terminated *** 2024-09-13T15:01:47.028 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:46 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:46.773+0000 7f3b8f0dc640 -1 osd.5 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:47.506 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:47.026+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:47.506 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:47.026+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:47.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:47.090+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:47.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:47.090+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:48.256 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:48.106+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:48.256 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:48.106+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:48.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:48.060+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:48.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:48.060+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:48.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:47.900+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:48.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:47 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:47.900+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:49.256 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:49.130+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:49.256 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:49.130+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:49.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:49.078+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:49.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:49.078+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:49.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:48.887+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:49.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:48 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:48.887+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:50.117 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:50.071+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:50.117 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:50.071+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:50.117 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:50.071+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:50.117 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:49.854+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:50.118 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:49.854+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:50.118 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:49 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:49.854+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:50.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:50.115+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:50.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:50.115+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:51.159 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:51.041+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:51.159 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:51.041+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:51.159 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:51.041+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:51.159 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:50.829+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:51.159 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:50.829+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:51.160 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:50 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:50.829+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:51.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:51.157+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:05.867949+0000 front 2024-09-13T15:01:05.867846+0000 (oldest deadline 2024-09-13T15:01:31.167896+0000) 2024-09-13T15:01:51.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5[172251]: 2024-09-13T15:01:51.157+0000 7f3b886ee640 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:15.868728+0000 front 2024-09-13T15:01:15.868778+0000 (oldest deadline 2024-09-13T15:01:42.269211+0000) 2024-09-13T15:01:52.167 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.037+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:52.168 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.037+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:52.168 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.037+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:52.168 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:51.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:52.168 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:51.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:52.168 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:51 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:51.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:52.485 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:52 smithi189 podman[268748]: 2024-09-13 15:01:52.166223283 +0000 UTC m=+5.508217938 container died c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T15:01:52.749 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:52 smithi189 podman[268748]: 2024-09-13 15:01:52.583542686 +0000 UTC m=+5.925537345 container cleanup c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T15:01:52.749 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:52 smithi189 bash[268748]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T15:01:53.006 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:52 smithi189 podman[268772]: 2024-09-13 15:01:52.835204853 +0000 UTC m=+0.665875104 container remove c5427ad361f81e7f4d88febc20cb9f7ce4e02c9eb55183b2a4f5334c5ab54fa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T15:01:53.006 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.997+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:53.006 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.997+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:53.007 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:52.997+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:53.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:52.748+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:53.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:52.748+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:53.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:52 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:52.748+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:53.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.244397706 +0000 UTC m=+0.110385945 container create b3e3ad1c3d404b0309149488d57204dcc261b4c16a3341d860b087fc3b578c15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:01:53.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.149891599 +0000 UTC m=+0.015879848 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:01:53.506 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.488586396 +0000 UTC m=+0.354574635 container init b3e3ad1c3d404b0309149488d57204dcc261b4c16a3341d860b087fc3b578c15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T15:01:53.507 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.49338263 +0000 UTC m=+0.359370870 container start b3e3ad1c3d404b0309149488d57204dcc261b4c16a3341d860b087fc3b578c15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T15:01:53.949 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.605337073 +0000 UTC m=+0.471325322 container attach b3e3ad1c3d404b0309149488d57204dcc261b4c16a3341d860b087fc3b578c15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=quincy-bbdadfa, io.buildah.version=1.37.2) 2024-09-13T15:01:53.949 INFO:journalctl@ceph.osd.5.smithi189.stdout:Sep 13 15:01:53 smithi189 podman[268862]: 2024-09-13 15:01:53.667937004 +0000 UTC m=+0.533925247 container died b3e3ad1c3d404b0309149488d57204dcc261b4c16a3341d860b087fc3b578c15 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-09-13T15:01:53.950 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:53.769+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:53.950 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:53.769+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:53.950 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:53.769+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:54.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:53.947+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:54.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:53.947+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:54.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:53 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:53.947+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:54.280 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T15:01:54.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:01:54.321 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T15:01:54.322 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T15:01:54.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.6 2024-09-13T15:01:54.649 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 systemd[1]: Stopping Ceph osd.6 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:01:54.960 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.647+0000 7fd5d1aeb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:01:54.960 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.647+0000 7fd5d1aeb640 -1 osd.6 518 *** Got signal Terminated *** 2024-09-13T15:01:54.960 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.647+0000 7fd5d1aeb640 -1 osd.6 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:01:54.960 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:54.961 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:54.961 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:54.735+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:54.961 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:54.735+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:54.961 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:54.735+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:55.255 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:54 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:54.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:56.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:55.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:56.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:55.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:56.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:55.958+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:56.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:55.757+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:56.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:55.757+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:56.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:55 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:55.757+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:56.980 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:56.727+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:56.981 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:56.727+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:56.981 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:56.727+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:57.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:56.979+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:57.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:56.979+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:57.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:56 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:56.979+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:58.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:57.749+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:58.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:57.749+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:58.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:57.749+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:58.007 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:57.972+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:58.007 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:57.972+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:58.007 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:57.972+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:58.007 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:57 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:57.972+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:33.650984+0000 front 2024-09-13T15:01:33.651093+0000 (oldest deadline 2024-09-13T15:01:57.651337+0000) 2024-09-13T15:01:59.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:58.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:01:59.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:58.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:01:59.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:58.792+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:01:59.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:58.928+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:08.047421+0000 front 2024-09-13T15:01:08.047641+0000 (oldest deadline 2024-09-13T15:01:31.447974+0000) 2024-09-13T15:01:59.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:58.928+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:19.049507+0000 front 2024-09-13T15:01:19.049592+0000 (oldest deadline 2024-09-13T15:01:44.949379+0000) 2024-09-13T15:01:59.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:58.928+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:24.949941+0000 front 2024-09-13T15:01:24.949841+0000 (oldest deadline 2024-09-13T15:01:49.550205+0000) 2024-09-13T15:01:59.257 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:01:58 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6[51161]: 2024-09-13T15:01:58.928+0000 7fd5cf105640 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:33.650984+0000 front 2024-09-13T15:01:33.651093+0000 (oldest deadline 2024-09-13T15:01:57.651337+0000) 2024-09-13T15:02:00.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:59.744+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:00.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:59.744+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:00.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:59.744+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:00.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:01:59 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:01:59.744+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:00.597 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:00 smithi189 podman[268989]: 2024-09-13 15:02:00.272947624 +0000 UTC m=+5.730387177 container died e712d52d9e990503de31f23bd58ffec2a9d7a1c97a1b6653534d86649037a4cd (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-bbdadfa) 2024-09-13T15:02:00.956 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:00 smithi189 podman[268989]: 2024-09-13 15:02:00.686860444 +0000 UTC m=+6.144299997 container cleanup e712d52d9e990503de31f23bd58ffec2a9d7a1c97a1b6653534d86649037a4cd (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T15:02:00.956 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:00 smithi189 bash[268989]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6 2024-09-13T15:02:00.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:00.781+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:00.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:00.781+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:00.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:00.781+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:00.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:00 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:00.781+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:01.256 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:00 smithi189 podman[269012]: 2024-09-13 15:02:00.955176102 +0000 UTC m=+0.658345834 container remove e712d52d9e990503de31f23bd58ffec2a9d7a1c97a1b6653534d86649037a4cd (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T15:02:01.601 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.168786485 +0000 UTC m=+0.016092115 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:02:01.601 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.289527823 +0000 UTC m=+0.136833477 container create c69b040578cce8a76a086fae97bcb20f740e7d6445eb0fb6cf688a7eb3c80b93 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-bbdadfa, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T15:02:01.601 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.500381134 +0000 UTC m=+0.347686755 container init c69b040578cce8a76a086fae97bcb20f740e7d6445eb0fb6cf688a7eb3c80b93 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T15:02:01.601 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.505575143 +0000 UTC m=+0.352880762 container start c69b040578cce8a76a086fae97bcb20f740e7d6445eb0fb6cf688a7eb3c80b93 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T15:02:01.602 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.600295628 +0000 UTC m=+0.447601263 container attach c69b040578cce8a76a086fae97bcb20f740e7d6445eb0fb6cf688a7eb3c80b93 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:02:01.979 INFO:journalctl@ceph.osd.6.smithi189.stdout:Sep 13 15:02:01 smithi189 podman[269103]: 2024-09-13 15:02:01.681960402 +0000 UTC m=+0.529266015 container died c69b040578cce8a76a086fae97bcb20f740e7d6445eb0fb6cf688a7eb3c80b93 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T15:02:01.980 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:01 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:01.755+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:01.980 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:01 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:01.755+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:01.980 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:01 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:01.755+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:01.980 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:01 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:01.755+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:02.281 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T15:02:02.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:02:02.321 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T15:02:02.321 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T15:02:02.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.7 2024-09-13T15:02:02.508 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 systemd[1]: Stopping Ceph osd.7 for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:03.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.650+0000 7f78b2a21640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:02:03.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.650+0000 7f78b2a21640 -1 osd.7 518 *** Got signal Terminated *** 2024-09-13T15:02:03.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.650+0000 7f78b2a21640 -1 osd.7 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:02:03.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.737+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:03.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.737+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:03.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.737+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:03.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:02 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:02.737+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:04.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:03.780+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:04.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:03.780+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:04.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:03.780+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:04.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:03 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:03.780+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:05.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:04.742+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:05.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:04.742+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:05.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:04.742+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:05.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:04 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:04.742+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:06.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:05.728+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:06.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:05.728+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:06.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:05.728+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:06.006 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:05.728+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:06.007 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:05 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:05.728+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-09-13T15:01:42.654641+0000 front 2024-09-13T15:01:42.654655+0000 (oldest deadline 2024-09-13T15:02:05.554617+0000) 2024-09-13T15:02:07.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:06.772+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-13T15:01:09.851855+0000 front 2024-09-13T15:01:09.851729+0000 (oldest deadline 2024-09-13T15:01:34.551564+0000) 2024-09-13T15:02:07.256 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:06.772+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-13T15:01:14.552031+0000 front 2024-09-13T15:01:14.552048+0000 (oldest deadline 2024-09-13T15:01:40.451997+0000) 2024-09-13T15:02:07.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:06.772+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-13T15:01:25.153032+0000 front 2024-09-13T15:01:25.153095+0000 (oldest deadline 2024-09-13T15:01:49.852799+0000) 2024-09-13T15:02:07.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:06.772+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-13T15:01:34.454085+0000 front 2024-09-13T15:01:34.454106+0000 (oldest deadline 2024-09-13T15:01:59.153911+0000) 2024-09-13T15:02:07.257 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:06 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7[57266]: 2024-09-13T15:02:06.772+0000 7f78ae037640 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-09-13T15:01:42.654641+0000 front 2024-09-13T15:01:42.654655+0000 (oldest deadline 2024-09-13T15:02:05.554617+0000) 2024-09-13T15:02:08.525 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:08 smithi189 podman[269234]: 2024-09-13 15:02:08.260008288 +0000 UTC m=+5.723566077 container died 35088803cdcaf2a394f23d72dcced79d6556567b9030a77b199133c533e84aca (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, RELEASE=quincy-bbdadfa, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T15:02:08.900 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:08 smithi189 podman[269234]: 2024-09-13 15:02:08.614838629 +0000 UTC m=+6.078396415 container cleanup 35088803cdcaf2a394f23d72dcced79d6556567b9030a77b199133c533e84aca (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-bbdadfa, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T15:02:08.900 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:08 smithi189 bash[269234]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7 2024-09-13T15:02:09.217 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:08 smithi189 podman[269268]: 2024-09-13 15:02:08.899436623 +0000 UTC m=+0.635895679 container remove 35088803cdcaf2a394f23d72dcced79d6556567b9030a77b199133c533e84aca (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, RELEASE=quincy-bbdadfa, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T15:02:09.506 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.216904901 +0000 UTC m=+0.110806409 container create 8adcc6d161bae8ffd0e7e10a970498a7703a0359ddab1253fa8c139b57d21153 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=quincy-bbdadfa, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T15:02:09.506 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.121950731 +0000 UTC m=+0.015852257 image pull a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T15:02:09.507 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.427869415 +0000 UTC m=+0.321770933 container init 8adcc6d161bae8ffd0e7e10a970498a7703a0359ddab1253fa8c139b57d21153 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T15:02:09.507 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.432885386 +0000 UTC m=+0.326786902 container start 8adcc6d161bae8ffd0e7e10a970498a7703a0359ddab1253fa8c139b57d21153 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-bbdadfa, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T15:02:09.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.524646401 +0000 UTC m=+0.418547913 container attach 8adcc6d161bae8ffd0e7e10a970498a7703a0359ddab1253fa8c139b57d21153 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=quincy-bbdadfa, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T15:02:09.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Sep 13 15:02:09 smithi189 podman[269361]: 2024-09-13 15:02:09.614305461 +0000 UTC m=+0.508206984 container died 8adcc6d161bae8ffd0e7e10a970498a7703a0359ddab1253fa8c139b57d21153 (image=quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-bbdadfa, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T15:02:10.258 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T15:02:10.296 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:02:10.297 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T15:02:10.297 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T15:02:10.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T15:02:10.506 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 systemd[1]: Stopping Ceph prometheus.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:10.918 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:02:10.918 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T15:02:10.919 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T15:02:10.919 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T15:02:10.919 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T15:02:10.919 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.628Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.631Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.631Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T15:02:10.920 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a[81233]: ts=2024-09-13T15:02:10.631Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T15:02:10.921 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:10 smithi189 podman[269489]: 2024-09-13 15:02:10.638990639 +0000 UTC m=+0.122341392 container died 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:11.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:11 smithi189 podman[269489]: 2024-09-13 15:02:11.017648999 +0000 UTC m=+0.500999751 container cleanup 770a1c26888c77189ad6eda64bac162e1586d21109dbb427e193dedaebd43b8b (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:11.256 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Sep 13 15:02:11 smithi189 bash[269489]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T15:02:11.417 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-523f52e4-71d9-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T15:02:11.451 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:02:11.452 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T15:02:11.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:02:13.433 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:13 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:13.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:13 smithi017 ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a[91497]: ts=2024-09-13T15:02:13.471Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:02:13.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:13 smithi017 podman[418030]: 2024-09-13 15:02:13.522446981 +0000 UTC m=+0.165831330 container died ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:14.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:13 smithi017 podman[418030]: 2024-09-13 15:02:13.988098555 +0000 UTC m=+0.631482945 container cleanup ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:14.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:13 smithi017 bash[418030]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T15:02:14.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:14 smithi017 podman[418051]: 2024-09-13 15:02:14.331629173 +0000 UTC m=+0.800856777 container remove ceb612349939f8cdf9e7c9552ad6ca3244371683d464069398469ea34c525c57 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:14.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:14 smithi017 podman[418051]: 2024-09-13 15:02:14.390840726 +0000 UTC m=+0.860068330 volume remove aee832fd16715068611db9b3fe7cf2dc57fd45b16b627440bfc51907b5861e20 2024-09-13T15:02:14.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:14 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T15:02:14.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:14 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:02:14.701 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 13 15:02:14 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 3.846s CPU time. 2024-09-13T15:02:14.982 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:14 smithi017 systemd[1]: Stopping Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:15.246 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:14 smithi017 podman[418162]: 2024-09-13 15:02:14.981663908 +0000 UTC m=+0.113109668 container died fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:15.595 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 podman[418162]: 2024-09-13 15:02:15.33506711 +0000 UTC m=+0.466512870 container cleanup fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:15.595 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 bash[418162]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T15:02:15.950 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 podman[418180]: 2024-09-13 15:02:15.594417198 +0000 UTC m=+0.616230710 container remove fce992870ef4aa26882a2d057ca7007de9b0c78555040e722d4ca83c61aaa9fe (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:02:15.951 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:02:15.951 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T15:02:15.951 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 systemd[1]: Stopped Ceph node-exporter.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:02:15.951 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 13 15:02:15 smithi017 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 8.620s CPU time. 2024-09-13T15:02:23.262 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:02:25.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:24 smithi189 systemd[1]: Stopping Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:25.300 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 podman[269906]: 2024-09-13 15:02:25.027170912 +0000 UTC m=+0.099710981 container died d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, version=8.5, architecture=x86_64, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897) 2024-09-13T15:02:25.717 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 podman[269906]: 2024-09-13 15:02:25.398396471 +0000 UTC m=+0.470936541 container cleanup d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, io.openshift.tags=base rhel8, io.openshift.expose-services=, version=8.5, architecture=x86_64, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Paul Cuzner , vendor=Red Hat, Inc., release=239.1651231664, vcs-type=git, description=Ceph Grafana Container, name=ubi8, build-date=2022-04-29T11:48:11.055897) 2024-09-13T15:02:25.717 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 bash[269906]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T15:02:26.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 podman[269926]: 2024-09-13 15:02:25.716292383 +0000 UTC m=+0.692088207 container remove d0c012ef79a6467faa6a7f2bd0979133e79b41b9330ee6515f8af19834944d00 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vendor=Red Hat, Inc., distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664) 2024-09-13T15:02:26.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:02:26.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T15:02:26.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 systemd[1]: Stopped Ceph grafana.a for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:02:26.006 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Sep 13 15:02:25 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@grafana.a.service: Consumed 28.876s CPU time. 2024-09-13T15:02:26.283 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 systemd[1]: Stopping Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:02:26.537 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 podman[270036]: 2024-09-13 15:02:26.282099645 +0000 UTC m=+0.111599295 container died 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:02:26.896 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 podman[270036]: 2024-09-13 15:02:26.63581054 +0000 UTC m=+0.465310190 container cleanup 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:02:26.896 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 bash[270036]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T15:02:27.256 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 podman[270053]: 2024-09-13 15:02:26.895349154 +0000 UTC m=+0.617076817 container remove 68ff3e2b1910ca3ee77bc9ec27b670776b6e8c9be0b00138808b9e9811aee7fc (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-523f52e4-71d9-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:02:27.256 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:26 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:02:27.256 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:27 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T15:02:27.256 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:27 smithi189 systemd[1]: Stopped Ceph node-exporter.b for 523f52e4-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:02:27.256 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Sep 13 15:02:27 smithi189 systemd[1]: ceph-523f52e4-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 7.743s CPU time. 2024-09-13T15:02:30.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:02:30.625 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:02:30.655 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T15:02:30.657 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi017/crash 2024-09-13T15:02:30.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:02:30.705 INFO:teuthology.orchestra.run.smithi017.stderr:tar: /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:02:30.705 INFO:teuthology.orchestra.run.smithi017.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:02:30.707 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi189/crash 2024-09-13T15:02:30.708 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:02:30.749 INFO:teuthology.orchestra.run.smithi189.stderr:tar: /var/lib/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:02:30.749 INFO:teuthology.orchestra.run.smithi189.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:02:30.749 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T15:02:30.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-09-13T15:02:30.837 INFO:teuthology.orchestra.run.smithi017.stdout:2024-09-13T14:34:20.344181+0000 mon.a (mon.0) 1040 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-13T15:02:30.837 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-13T15:02:30.838 INFO:tasks.cephadm:Compressing logs... 2024-09-13T15:02:30.838 DEBUG:teuthology.orchestra.run.smithi017:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:02:30.841 DEBUG:teuthology.orchestra.run.smithi189:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:02:30.876 INFO:teuthology.orchestra.run.smithi017.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T15:02:30.876 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T15:02:30.876 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T15:02:30.883 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:02:30.883 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:02:30.884 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi017.ldrmqj.log 2024-09-13T15:02:30.885 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log: /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T15:02:30.885 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi017.ldrmqj.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:02:30.885 INFO:teuthology.orchestra.run.smithi017.stderr: 82.0% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi017.ldrmqj.log.gz 2024-09-13T15:02:30.886 INFO:teuthology.orchestra.run.smithi189.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T15:02:30.886 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi189.ndtccr.log 2024-09-13T15:02:30.886 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T15:02:30.886 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi189.ndtccr.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr: 62.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi189.ndtccr.log.gz 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:02:30.888 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T15:02:30.888 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T15:02:30.888 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:02:30.889 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.6% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:02:31.118 INFO:teuthology.orchestra.run.smithi017.stderr: 91.1% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:02:31.146 INFO:teuthology.orchestra.run.smithi017.stderr: 95.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:02:31.204 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 84.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:02:31.265 INFO:teuthology.orchestra.run.smithi189.stderr: 86.8% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:02:31.289 INFO:teuthology.orchestra.run.smithi189.stderr: 89.9% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:02:31.349 INFO:teuthology.orchestra.run.smithi017.stderr: 91.0% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:02:31.402 INFO:teuthology.orchestra.run.smithi017.stderr: 93.9% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:02:31.524 INFO:teuthology.orchestra.run.smithi189.stderr: 91.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:02:31.696 INFO:teuthology.orchestra.run.smithi189.stderr: 90.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T15:02:31.705 INFO:teuthology.orchestra.run.smithi189.stderr: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:02:34.012 INFO:teuthology.orchestra.run.smithi017.stderr: 89.5% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T15:02:53.334 INFO:teuthology.orchestra.run.smithi189.stderr: 92.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T15:02:54.618 INFO:teuthology.orchestra.run.smithi017.stderr: 91.4% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T15:03:11.843 INFO:teuthology.orchestra.run.smithi017.stderr: 90.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T15:05:34.201 INFO:teuthology.orchestra.run.smithi189.stderr: 92.6% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T15:08:17.027 INFO:teuthology.orchestra.run.smithi189.stderr: 93.3% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T15:09:19.899 INFO:teuthology.orchestra.run.smithi189.stderr: 93.6% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T15:09:27.511 INFO:teuthology.orchestra.run.smithi189.stderr: 93.2% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T15:09:27.513 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-09-13T15:09:27.514 INFO:teuthology.orchestra.run.smithi189.stderr:real 6m56.657s 2024-09-13T15:09:27.514 INFO:teuthology.orchestra.run.smithi189.stderr:user 5m36.868s 2024-09-13T15:09:27.514 INFO:teuthology.orchestra.run.smithi189.stderr:sys 0m15.643s 2024-09-13T15:25:40.422 INFO:teuthology.orchestra.run.smithi017.stderr: 93.0% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T15:25:44.190 INFO:teuthology.orchestra.run.smithi017.stderr: 92.9% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T15:26:29.574 INFO:teuthology.orchestra.run.smithi017.stderr: 93.1% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T15:26:44.137 INFO:teuthology.orchestra.run.smithi017.stderr: 92.7% -- replaced with /var/log/ceph/523f52e4-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T15:26:44.179 INFO:teuthology.orchestra.run.smithi017.stderr: 2024-09-13T15:26:44.179 INFO:teuthology.orchestra.run.smithi017.stderr:real 24m13.325s 2024-09-13T15:26:44.179 INFO:teuthology.orchestra.run.smithi017.stderr:user 8m44.660s 2024-09-13T15:26:44.180 INFO:teuthology.orchestra.run.smithi017.stderr:sys 0m25.058s 2024-09-13T15:26:44.180 INFO:tasks.cephadm:Archiving logs... 2024-09-13T15:26:44.180 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi017/log 2024-09-13T15:26:44.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:28:01.287 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi189/log 2024-09-13T15:28:01.289 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:28:56.596 INFO:tasks.cephadm:Removing cluster... 2024-09-13T15:28:56.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T15:28:58.115 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 523f52e4-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T15:28:59.076 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T15:28:59.076 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:28:59.110 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:28:59.126 INFO:tasks.cephadm:Teardown complete 2024-09-13T15:28:59.126 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T15:28:59.137 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T15:28:59.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:28:59.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:28:59.261 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T15:28:59.271 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T15:28:59.271 DEBUG:teuthology.orchestra.run.smithi017:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:28:59.274 DEBUG:teuthology.orchestra.run.smithi189:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:28:59.288 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:28:59.289 INFO:teuthology.orchestra.run.smithi017.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:28:59.305 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:28:59.305 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-09-13T15:28:59.305 INFO:teuthology.orchestra.run.smithi189.stdout:^* hv01.front.sepia.ceph.com 3 6 377 5 +341us[ +340us] +/- 59ms 2024-09-13T15:28:59.306 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 8 -854us[ -855us] +/- 74ms 2024-09-13T15:28:59.306 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 7 +476us[ +475us] +/- 74ms 2024-09-13T15:28:59.306 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:^* hv01.front.sepia.ceph.com 3 6 377 7 +343us[ +344us] +/- 59ms 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 73 -806us[ -815us] +/- 73ms 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 3 +551us[ +551us] +/- 74ms 2024-09-13T15:28:59.410 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:28:59.422 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T15:28:59.432 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T15:28:59.433 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T15:28:59.447 DEBUG:teuthology.orchestra.run.smithi017:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:28:59.627 DEBUG:teuthology.orchestra.run.smithi189:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:28:59.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T15:28:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:28:59.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T15:28:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:28:59.871 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T15:28:59.881 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T15:28:59.893 INFO:teuthology.task.internal:Duration was 5352.343528 seconds 2024-09-13T15:28:59.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T15:28:59.911 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T15:28:59.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:28:59.914 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:28:59.965 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:28:59.985 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:29:00.314 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T15:29:00.314 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi017.front.sepia.ceph.com 2024-09-13T15:29:00.314 DEBUG:teuthology.orchestra.run.smithi017:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:29:00.345 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi189.front.sepia.ceph.com 2024-09-13T15:29:00.345 DEBUG:teuthology.orchestra.run.smithi189:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:29:00.374 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T15:29:00.374 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:29:00.388 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:29:00.441 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T15:29:00.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:29:00.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:29:02.469 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T15:29:02.479 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T15:29:02.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:29:02.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:29:02.552 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T15:29:02.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:29:02.569 DEBUG:teuthology.orchestra.run.smithi189:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:29:02.606 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = core 2024-09-13T15:29:02.625 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = core 2024-09-13T15:29:02.653 DEBUG:teuthology.orchestra.run.smithi017:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:29:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:29:02.708 DEBUG:teuthology.orchestra.run.smithi189:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:29:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:29:02.723 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T15:29:02.734 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T15:29:02.735 DEBUG:teuthology.misc:Transferring archived files from smithi017:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi017 2024-09-13T15:29:02.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:29:02.832 DEBUG:teuthology.misc:Transferring archived files from smithi189:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903542/remote/smithi189 2024-09-13T15:29:02.832 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:29:02.881 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T15:29:02.882 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:29:02.884 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:29:02.926 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T15:29:02.937 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T15:29:02.937 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T15:29:02.947 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T15:29:02.947 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:29:02.949 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:29:02.970 INFO:teuthology.orchestra.run.smithi017.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:29 /home/ubuntu/cephtest 2024-09-13T15:29:02.970 INFO:teuthology.orchestra.run.smithi017.stdout: 527343 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 14:58 /home/ubuntu/cephtest/mnt.0 2024-09-13T15:29:02.971 INFO:teuthology.orchestra.run.smithi017.stdout: 527344 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 14:58 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T15:29:02.971 INFO:teuthology.orchestra.run.smithi017.stdout: 527385 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 14:58 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T15:29:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:29:02.972 INFO:teuthology.orchestra.run.smithi017.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T15:29:02.972 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:29:02.979 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T15:29:02.989 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T15:29:03.000 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:29:03.003 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 5352.343527793884 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_akupczyk@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=83878f5762b540d0b642848dc5bc46e6 status: fail success: false 2024-09-13T15:29:03.004 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T15:29:03.027 INFO:teuthology.orchestra.run.smithi189.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:29 /home/ubuntu/cephtest 2024-09-13T15:29:03.126 INFO:teuthology.run:FAIL